databased.databased
1import logging 2import os 3import sqlite3 4from datetime import datetime 5from functools import wraps 6from typing import Any 7 8import pandas 9from pathier import Pathier 10from tabulate import tabulate 11 12 13def _connect(func): 14 """Decorator to open db connection if it isn't already open.""" 15 16 @wraps(func) 17 def inner(self, *args, **kwargs): 18 if not self.connection_open: 19 self.open() 20 results = func(self, *args, **kwargs) 21 return results 22 23 return inner 24 25 26def _disconnect(func): 27 """Decorator to commit and close db connection. 28 29 Primarily intended for when `DataBased` is subclassed and the inhereting class 30 has functions that call parent class functions that are decorated with `_connect`. 31 Decorating the child class function with `_disconnect` avoids having to manually close 32 the connection or use a context manager in an application.""" 33 34 @wraps(func) 35 def inner(self, *args, **kwargs): 36 result = func(self, *args, **kwargs) 37 if self.connection_open: 38 self.close() 39 return result 40 41 return inner 42 43 44class DataBased: 45 """Sqli wrapper so queries don't need to be written except table definitions. 46 47 Supports saving and reading dates as datetime objects. 48 49 Supports using a context manager.""" 50 51 def __init__( 52 self, 53 dbpath: str | Pathier, 54 logger_encoding: str = "utf-8", 55 logger_message_format: str = "{levelname}|-|{asctime}|-|{message}", 56 connection_timeout: float = 10, 57 ): 58 """ 59 #### :params: 60 61 * `dbpath`: String or Path object to database file. 62 If a relative path is given, it will be relative to the 63 current working directory. The log file will be saved to the 64 same directory. 65 66 * `logger_message_format`: `{` style format string for the logger object. 67 68 * `connection_timeout`: The number of seconds to wait when trying to connect to the database before throwing an error.""" 69 self.dbpath = Pathier(dbpath) 70 self.dbname = Pathier(dbpath).name 71 self.dbpath.parent.mkdir(parents=True, exist_ok=True) 72 self._logger_init( 73 encoding=logger_encoding, message_format=logger_message_format 74 ) 75 self.connection_open = False 76 self.connection_timeout = connection_timeout 77 78 def __enter__(self): 79 self.open() 80 return self 81 82 def __exit__(self, exception_type, exception_value, exception_traceback): 83 self.close() 84 85 @property 86 def connection_timeout(self) -> float: 87 return self._connection_timeout 88 89 @connection_timeout.setter 90 def connection_timeout(self, num_seconds: float): 91 self._connection_timeout = num_seconds 92 if self.connection_open: 93 self.close() 94 self.open() 95 96 def open(self): 97 """Open connection to db.""" 98 self.connection = sqlite3.connect( 99 self.dbpath, 100 detect_types=sqlite3.PARSE_DECLTYPES | sqlite3.PARSE_COLNAMES, 101 timeout=self.connection_timeout, 102 ) 103 self.connection.execute("pragma foreign_keys = 1;") 104 self.cursor = self.connection.cursor() 105 self.connection_open = True 106 107 def close(self): 108 """Save and close connection to db. 109 110 Call this as soon as you are done using the database if you have 111 multiple threads or processes using the same database.""" 112 if self.connection_open: 113 self.connection.commit() 114 self.connection.close() 115 self.connection_open = False 116 117 def _logger_init( 118 self, 119 message_format: str = "{levelname}|-|{asctime}|-|{message}", 120 encoding: str = "utf-8", 121 ): 122 """:param `message_format`: '{' style format string""" 123 self.logger = logging.getLogger(self.dbname) 124 if not self.logger.hasHandlers(): 125 handler = logging.FileHandler( 126 str(self.dbpath).replace(".", "") + ".log", encoding=encoding 127 ) 128 handler.setFormatter( 129 logging.Formatter( 130 message_format, style="{", datefmt="%m/%d/%Y %I:%M:%S %p" 131 ) 132 ) 133 self.logger.addHandler(handler) 134 self.logger.setLevel(logging.INFO) 135 136 def _get_dict( 137 self, table: str, values: list, columns_to_return: list[str] | None = None 138 ) -> dict: 139 """Converts the values of a row into a dictionary with column names as keys. 140 141 #### :params: 142 143 `table`: The table that values were pulled from. 144 145 `values`: List of values expected to be the same quantity 146 and in the same order as the column names of table. 147 148 `columns_to_return`: An optional list of column names. 149 If given, only these columns will be included in the returned dictionary. 150 Otherwise all columns and values are returned.""" 151 return { 152 column: value 153 for column, value in zip(self.get_column_names(table), values) 154 if not columns_to_return or column in columns_to_return 155 } 156 157 def _get_conditions( 158 self, match_criteria: list[tuple] | dict, exact_match: bool = True 159 ) -> str: 160 """Builds and returns the conditional portion of a query. 161 162 #### :params: 163 164 `match_criteria`: Can be a list of 2-tuples where each 165 tuple is `(columnName, rowValue)` or a dictionary where 166 keys are column names and values are row values. 167 168 `exact_match`: If `False`, the row value for a given column 169 will be matched as a substring. 170 171 Usage e.g.: 172 173 >>> self.cursor.execute(f'select * from {table} where {conditions};')""" 174 if type(match_criteria) == dict: 175 match_criteria = [(k, v) for k, v in match_criteria.items()] 176 if exact_match: 177 conditions = " and ".join( 178 f'"{column_row[0]}" = "{column_row[1]}"' 179 for column_row in match_criteria 180 ) 181 else: 182 conditions = " and ".join( 183 f'"{column_row[0]}" like "%{column_row[1]}%"' 184 for column_row in match_criteria 185 ) 186 return f"({conditions})" 187 188 def vacuum(self): 189 """Reduce disk size of the database with a `VACUUM` query.""" 190 self.query("VACUUM") 191 192 @_connect 193 def query(self, query_) -> list[Any]: 194 """Execute an arbitrary query and return the results.""" 195 self.cursor.execute(query_) 196 return self.cursor.fetchall() 197 198 @_connect 199 def create_tables(self, table_defs: list[str] = []): 200 """Create tables if they don't exist. 201 202 :param `table_defs`: Each definition should be in the form `table_name(column_definitions)`""" 203 if len(table_defs) > 0: 204 table_names = self.get_table_names() 205 for table in table_defs: 206 if table.split("(")[0].strip() not in table_names: 207 self.cursor.execute(f"create table [{table}];") 208 self.logger.info(f'{table.split("(")[0]} table created.') 209 210 @_connect 211 def create_table(self, table: str, column_defs: list[str]): 212 """Create a table if it doesn't exist. 213 214 #### :params: 215 216 `table`: Name of the table to create. 217 218 `column_defs`: List of column definitions in proper Sqlite3 sytax. 219 i.e. `"column_name text unique"` or `"column_name int primary key"` etc.""" 220 if table not in self.get_table_names(): 221 query = f"create table [{table}]({', '.join(column_defs)});" 222 self.cursor.execute(query) 223 self.logger.info(f"'{table}' table created.") 224 225 @_connect 226 def get_table_names(self) -> list[str]: 227 """Returns a list of table names from the database.""" 228 self.cursor.execute( 229 'select name from sqlite_Schema where type = "table" and name not like "sqlite_%";' 230 ) 231 return [result[0] for result in self.cursor.fetchall()] 232 233 @_connect 234 def get_column_names(self, table: str) -> list[str]: 235 """Return a list of column names from a table.""" 236 self.cursor.execute(f"select * from [{table}] where 1=0;") 237 return [description[0] for description in self.cursor.description] 238 239 @_connect 240 def count( 241 self, 242 table: str, 243 match_criteria: list[tuple] | dict | None = None, 244 exact_match: bool = True, 245 ) -> int: 246 """Return number of items in `table`. 247 248 #### :params: 249 250 `match_criteria`: Can be a list of 2-tuples where each 251 tuple is `(columnName, rowValue)` or a dictionary where 252 keys are column names and values are row values. 253 If `None`, all rows from the table will be counted. 254 255 `exact_match`: If `False`, the row value for a given column 256 in `match_criteria` will be matched as a substring. 257 Has no effect if `match_criteria` is `None`. 258 """ 259 query = f"select count(_rowid_) from [{table}]" 260 try: 261 if match_criteria: 262 self.cursor.execute( 263 f"{query} where {self._get_conditions(match_criteria, exact_match)};" 264 ) 265 else: 266 self.cursor.execute(f"{query}") 267 return self.cursor.fetchone()[0] 268 except: 269 return 0 270 271 @_connect 272 def add_row( 273 self, table: str, values: tuple[Any], columns: tuple[str] | None = None 274 ) -> bool: 275 """Add a row of values to a table. 276 277 Returns whether the addition was successful or not. 278 279 #### :params: 280 281 `table`: The table to insert values into. 282 283 `values`: A tuple of values to be inserted into the table. 284 285 `columns`: If `None`, `values` is expected to supply a value for every column in the table. 286 If `columns` is provided, it should contain the same number of elements as `values`.""" 287 parameterizer = ", ".join("?" for _ in values) 288 logger_values = ", ".join(str(value) for value in values) 289 try: 290 if columns: 291 columns_query = ", ".join(column for column in columns) 292 self.cursor.execute( 293 f"insert into [{table}] ({columns_query}) values({parameterizer});", 294 values, 295 ) 296 else: 297 self.cursor.execute( 298 f"insert into [{table}] values({parameterizer});", values 299 ) 300 self.logger.info(f'Added "{logger_values}" to {table} table.') 301 return True 302 except Exception as e: 303 if "constraint" not in str(e).lower(): 304 self.logger.exception( 305 f'Error adding "{logger_values}" to {table} table.' 306 ) 307 else: 308 self.logger.debug(str(e)) 309 return False 310 311 @_connect 312 def add_rows( 313 self, table: str, values: list[tuple[Any]], columns: tuple[str] | None = None 314 ) -> tuple[int, int]: 315 """Add multiple rows of values to a table. 316 317 Returns a tuple containing the number of successful additions and the number of failed additions. 318 319 #### :params: 320 321 `table`: The table to insert values into. 322 323 `values`: A list of tuples of values to be inserted into the table. 324 Each tuple constitutes a single row to be inserted 325 326 `columns`: If `None`, `values` is expected to supply a value for every column in the table. 327 If `columns` is provided, it should contain the same number of elements as `values`.""" 328 successes = 0 329 failures = 0 330 for row in values: 331 if self.add_row(table, row, columns): 332 successes += 1 333 else: 334 failures += 1 335 return (successes, failures) 336 337 @_connect 338 def get_rows( 339 self, 340 table: str, 341 match_criteria: list[tuple] | dict | None = None, 342 exact_match: bool = True, 343 sort_by_column: str | None = None, 344 columns_to_return: list[str] | None = None, 345 return_as_dataframe: bool = False, 346 values_only: bool = False, 347 order_by: str | None = None, 348 limit: str | int | None = None, 349 ) -> list[dict] | list[tuple] | pandas.DataFrame: 350 """Return matching rows from `table`. 351 352 By default, rows will be returned as a list of dictionaries of the form `[{"column_name": value, ...}, ...]` 353 354 355 #### :params: 356 357 `match_criteria`: Can be a list of 2-tuples where each 358 tuple is `(columnName, rowValue)` or a dictionary where 359 keys are column names and values are row values. 360 361 `exact_match`: If `False`, the row value for a given column will be matched as a substring. 362 363 `sort_by_column`: A column name to sort the results by. 364 This will sort results in Python after retrieving them from the db. 365 Use the 'order_by' param to use SQLite engine for ordering. 366 367 `columns_to_return`: Optional list of column names. 368 If provided, the elements returned by this function will only contain the provided columns. 369 Otherwise every column in the row is returned. 370 371 `return_as_dataframe`: Return the results as a `pandas.DataFrame` object. 372 373 `values_only`: Return the results as a list of tuples. 374 375 `order_by`: If given, a `order by {order_by}` clause will be added to the select query. 376 377 `limit`: If given, a `limit {limit}` clause will be added to the select query. 378 """ 379 380 if type(columns_to_return) is str: 381 columns_to_return = [columns_to_return] 382 query = f"select * from [{table}]" 383 matches = [] 384 if match_criteria: 385 query += f" where {self._get_conditions(match_criteria, exact_match)}" 386 if order_by: 387 query += f" order by {order_by}" 388 if limit: 389 query += f" limit {limit}" 390 query += ";" 391 self.cursor.execute(query) 392 matches = self.cursor.fetchall() 393 results = [self._get_dict(table, match, columns_to_return) for match in matches] 394 if sort_by_column: 395 results = sorted(results, key=lambda x: x[sort_by_column]) 396 if return_as_dataframe: 397 return pandas.DataFrame(results) 398 if values_only: 399 return [tuple(row.values()) for row in results] 400 else: 401 return results 402 403 @_connect 404 def find( 405 self, table: str, query_string: str, columns: list[str] | None = None 406 ) -> list[dict]: 407 """Search for rows that contain `query_string` as a substring of any column. 408 409 #### :params: 410 411 `table`: The table to search. 412 413 `query_string`: The substring to search for in all columns. 414 415 `columns`: A list of columns to search for query_string. 416 If None, all columns in the table will be searched. 417 """ 418 if type(columns) is str: 419 columns = [columns] 420 results = [] 421 if not columns: 422 columns = self.get_column_names(table) 423 for column in columns: 424 results.extend( 425 [ 426 row 427 for row in self.get_rows( 428 table, [(column, query_string)], exact_match=False 429 ) 430 if row not in results 431 ] 432 ) 433 return results 434 435 @_connect 436 def delete( 437 self, table: str, match_criteria: list[tuple] | dict, exact_match: bool = True 438 ) -> int: 439 """Delete records from `table`. 440 441 Returns the number of deleted records. 442 443 #### :params: 444 445 `match_criteria`: Can be a list of 2-tuples where each tuple is `(column_name, value)` 446 or a dictionary where keys are column names and values are corresponding values. 447 448 `exact_match`: If `False`, the value for a given column will be matched as a substring. 449 """ 450 conditions = self._get_conditions(match_criteria, exact_match) 451 try: 452 self.cursor.execute(f"delete from [{table}] where {conditions};") 453 num_deletions = self.cursor.rowcount 454 self.logger.info( 455 f'Deleted {num_deletions} rows from "{table}" where {conditions}".' 456 ) 457 return num_deletions 458 except Exception as e: 459 self.logger.debug( 460 f'Error deleting rows from "{table}" where {conditions}.\n{e}' 461 ) 462 return 0 463 464 @_connect 465 def update( 466 self, 467 table: str, 468 column_to_update: str, 469 new_value: Any, 470 match_criteria: list[tuple] | dict | None = None, 471 exact_match: bool = True, 472 ) -> int: 473 """Update the value in `column_to_update` to `new_value` for rows matched with `match_criteria`. 474 475 #### :params: 476 477 `table`: The table to update rows in. 478 479 `column_to_update`: The column to be updated in the matched rows. 480 481 `new_value`: The new value to insert. 482 483 `match_criteria`: Can be a list of 2-tuples where each tuple is `(columnName, rowValue)` 484 or a dictionary where keys are column names and values are corresponding values. 485 If `None`, every row in `table` will be updated. 486 487 `exact_match`: If `False`, `match_criteria` values will be treated as substrings. 488 489 Returns the number of updated rows.""" 490 query = f"update [{table}] set {column_to_update} = ?" 491 conditions = "" 492 if match_criteria: 493 conditions = self._get_conditions(match_criteria, exact_match) 494 query += f" where {conditions}" 495 else: 496 conditions = None 497 query += ";" 498 try: 499 self.cursor.execute( 500 query, 501 (new_value,), 502 ) 503 num_updates = self.cursor.rowcount 504 self.logger.info( 505 f'In {num_updates} rows, updated "{column_to_update}" in "{table}" table to "{new_value}" where {conditions}' 506 ) 507 return num_updates 508 except Exception as e: 509 self.logger.error( 510 f'Failed to update "{column_to_update}" in "{table}" table to "{new_value}" where {conditions}"\n{e}' 511 ) 512 return 0 513 514 @_connect 515 def drop_table(self, table: str) -> bool: 516 """Drop `table` from the database. 517 518 Returns `True` if successful, `False` if not.""" 519 try: 520 self.cursor.execute(f"drop Table [{table}];") 521 self.logger.info(f'Dropped table "{table}"') 522 return True 523 except Exception as e: 524 print(e) 525 self.logger.error(f'Failed to drop table "{table}"') 526 return False 527 528 @_connect 529 def add_column( 530 self, table: str, column: str, _type: str, default_value: str | None = None 531 ): 532 """Add a new column to `table`. 533 534 #### :params: 535 536 `column`: Name of the column to add. 537 538 `_type`: The data type of the new column. 539 540 `default_value`: Optional default value for the column.""" 541 try: 542 if default_value: 543 self.cursor.execute( 544 f"alter table [{table}] add column {column} {_type} default {default_value};" 545 ) 546 self.update(table, column, default_value) 547 else: 548 self.cursor.execute( 549 f"alter table [{table}] add column {column} {_type};" 550 ) 551 self.logger.info(f'Added column "{column}" to "{table}" table.') 552 except Exception as e: 553 self.logger.error(f'Failed to add column "{column}" to "{table}" table.') 554 555 @staticmethod 556 def data_to_string( 557 data: list[dict], sort_key: str | None = None, wrap_to_terminal: bool = True 558 ) -> str: 559 """Uses tabulate to produce pretty string output from a list of dictionaries. 560 561 #### :params: 562 563 `data`: The list of dictionaries to create a grid from. 564 Assumes all dictionaries in list have the same set of keys. 565 566 `sort_key`: Optional dictionary key to sort data with. 567 568 `wrap_to_terminal`: If `True`, the table width will be wrapped to fit within the current terminal window. 569 Pass as `False` if the output is going into something like a `.txt` file.""" 570 return data_to_string(data, sort_key, wrap_to_terminal) 571 572 573def data_to_string( 574 data: list[dict], sort_key: str | None = None, wrap_to_terminal: bool = True 575) -> str: 576 """Uses tabulate to produce pretty string output from a list of dictionaries. 577 578 #### :params: 579 580 `data`: The list of dictionaries to create a grid from. 581 Assumes all dictionaries in list have the same set of keys. 582 583 `sort_key`: Optional dictionary key to sort data with. 584 585 `wrap_to_terminal`: If `True`, the table width will be wrapped to fit within the current terminal window. 586 Pass as `False` if the output is going into something like a `.txt` file.""" 587 if len(data) == 0: 588 return "" 589 if sort_key: 590 data = sorted(data, key=lambda d: d[sort_key]) 591 for i, d in enumerate(data): 592 for k in d: 593 data[i][k] = str(data[i][k]) 594 595 too_wide = True 596 terminal_width = os.get_terminal_size().columns 597 max_col_widths = terminal_width 598 # Make an output with effectively unrestricted column widths 599 # to see if shrinking is necessary 600 output = tabulate( 601 data, 602 headers="keys", 603 disable_numparse=True, 604 tablefmt="grid", 605 maxcolwidths=max_col_widths, 606 ) 607 current_width = output.index("\n") 608 if current_width < terminal_width: 609 too_wide = False 610 if wrap_to_terminal and too_wide: 611 print("Resizing grid to fit within the terminal...\n") 612 previous_col_widths = max_col_widths 613 acceptable_width = terminal_width - 10 614 while too_wide and max_col_widths > 1: 615 if current_width >= terminal_width: 616 previous_col_widths = max_col_widths 617 max_col_widths = int(max_col_widths * 0.5) 618 elif current_width < terminal_width: 619 # Without lowering acceptable_width, this condition will cause infinite loop 620 if max_col_widths == previous_col_widths - 1: 621 acceptable_width -= 10 622 max_col_widths = int( 623 max_col_widths + (0.5 * (previous_col_widths - max_col_widths)) 624 ) 625 output = tabulate( 626 data, 627 headers="keys", 628 disable_numparse=True, 629 tablefmt="grid", 630 maxcolwidths=max_col_widths, 631 ) 632 current_width = output.index("\n") 633 if acceptable_width < current_width < terminal_width: 634 too_wide = False 635 if too_wide: 636 print("Couldn't resize grid to fit within the terminal.") 637 return str(data) 638 return output
45class DataBased: 46 """Sqli wrapper so queries don't need to be written except table definitions. 47 48 Supports saving and reading dates as datetime objects. 49 50 Supports using a context manager.""" 51 52 def __init__( 53 self, 54 dbpath: str | Pathier, 55 logger_encoding: str = "utf-8", 56 logger_message_format: str = "{levelname}|-|{asctime}|-|{message}", 57 connection_timeout: float = 10, 58 ): 59 """ 60 #### :params: 61 62 * `dbpath`: String or Path object to database file. 63 If a relative path is given, it will be relative to the 64 current working directory. The log file will be saved to the 65 same directory. 66 67 * `logger_message_format`: `{` style format string for the logger object. 68 69 * `connection_timeout`: The number of seconds to wait when trying to connect to the database before throwing an error.""" 70 self.dbpath = Pathier(dbpath) 71 self.dbname = Pathier(dbpath).name 72 self.dbpath.parent.mkdir(parents=True, exist_ok=True) 73 self._logger_init( 74 encoding=logger_encoding, message_format=logger_message_format 75 ) 76 self.connection_open = False 77 self.connection_timeout = connection_timeout 78 79 def __enter__(self): 80 self.open() 81 return self 82 83 def __exit__(self, exception_type, exception_value, exception_traceback): 84 self.close() 85 86 @property 87 def connection_timeout(self) -> float: 88 return self._connection_timeout 89 90 @connection_timeout.setter 91 def connection_timeout(self, num_seconds: float): 92 self._connection_timeout = num_seconds 93 if self.connection_open: 94 self.close() 95 self.open() 96 97 def open(self): 98 """Open connection to db.""" 99 self.connection = sqlite3.connect( 100 self.dbpath, 101 detect_types=sqlite3.PARSE_DECLTYPES | sqlite3.PARSE_COLNAMES, 102 timeout=self.connection_timeout, 103 ) 104 self.connection.execute("pragma foreign_keys = 1;") 105 self.cursor = self.connection.cursor() 106 self.connection_open = True 107 108 def close(self): 109 """Save and close connection to db. 110 111 Call this as soon as you are done using the database if you have 112 multiple threads or processes using the same database.""" 113 if self.connection_open: 114 self.connection.commit() 115 self.connection.close() 116 self.connection_open = False 117 118 def _logger_init( 119 self, 120 message_format: str = "{levelname}|-|{asctime}|-|{message}", 121 encoding: str = "utf-8", 122 ): 123 """:param `message_format`: '{' style format string""" 124 self.logger = logging.getLogger(self.dbname) 125 if not self.logger.hasHandlers(): 126 handler = logging.FileHandler( 127 str(self.dbpath).replace(".", "") + ".log", encoding=encoding 128 ) 129 handler.setFormatter( 130 logging.Formatter( 131 message_format, style="{", datefmt="%m/%d/%Y %I:%M:%S %p" 132 ) 133 ) 134 self.logger.addHandler(handler) 135 self.logger.setLevel(logging.INFO) 136 137 def _get_dict( 138 self, table: str, values: list, columns_to_return: list[str] | None = None 139 ) -> dict: 140 """Converts the values of a row into a dictionary with column names as keys. 141 142 #### :params: 143 144 `table`: The table that values were pulled from. 145 146 `values`: List of values expected to be the same quantity 147 and in the same order as the column names of table. 148 149 `columns_to_return`: An optional list of column names. 150 If given, only these columns will be included in the returned dictionary. 151 Otherwise all columns and values are returned.""" 152 return { 153 column: value 154 for column, value in zip(self.get_column_names(table), values) 155 if not columns_to_return or column in columns_to_return 156 } 157 158 def _get_conditions( 159 self, match_criteria: list[tuple] | dict, exact_match: bool = True 160 ) -> str: 161 """Builds and returns the conditional portion of a query. 162 163 #### :params: 164 165 `match_criteria`: Can be a list of 2-tuples where each 166 tuple is `(columnName, rowValue)` or a dictionary where 167 keys are column names and values are row values. 168 169 `exact_match`: If `False`, the row value for a given column 170 will be matched as a substring. 171 172 Usage e.g.: 173 174 >>> self.cursor.execute(f'select * from {table} where {conditions};')""" 175 if type(match_criteria) == dict: 176 match_criteria = [(k, v) for k, v in match_criteria.items()] 177 if exact_match: 178 conditions = " and ".join( 179 f'"{column_row[0]}" = "{column_row[1]}"' 180 for column_row in match_criteria 181 ) 182 else: 183 conditions = " and ".join( 184 f'"{column_row[0]}" like "%{column_row[1]}%"' 185 for column_row in match_criteria 186 ) 187 return f"({conditions})" 188 189 def vacuum(self): 190 """Reduce disk size of the database with a `VACUUM` query.""" 191 self.query("VACUUM") 192 193 @_connect 194 def query(self, query_) -> list[Any]: 195 """Execute an arbitrary query and return the results.""" 196 self.cursor.execute(query_) 197 return self.cursor.fetchall() 198 199 @_connect 200 def create_tables(self, table_defs: list[str] = []): 201 """Create tables if they don't exist. 202 203 :param `table_defs`: Each definition should be in the form `table_name(column_definitions)`""" 204 if len(table_defs) > 0: 205 table_names = self.get_table_names() 206 for table in table_defs: 207 if table.split("(")[0].strip() not in table_names: 208 self.cursor.execute(f"create table [{table}];") 209 self.logger.info(f'{table.split("(")[0]} table created.') 210 211 @_connect 212 def create_table(self, table: str, column_defs: list[str]): 213 """Create a table if it doesn't exist. 214 215 #### :params: 216 217 `table`: Name of the table to create. 218 219 `column_defs`: List of column definitions in proper Sqlite3 sytax. 220 i.e. `"column_name text unique"` or `"column_name int primary key"` etc.""" 221 if table not in self.get_table_names(): 222 query = f"create table [{table}]({', '.join(column_defs)});" 223 self.cursor.execute(query) 224 self.logger.info(f"'{table}' table created.") 225 226 @_connect 227 def get_table_names(self) -> list[str]: 228 """Returns a list of table names from the database.""" 229 self.cursor.execute( 230 'select name from sqlite_Schema where type = "table" and name not like "sqlite_%";' 231 ) 232 return [result[0] for result in self.cursor.fetchall()] 233 234 @_connect 235 def get_column_names(self, table: str) -> list[str]: 236 """Return a list of column names from a table.""" 237 self.cursor.execute(f"select * from [{table}] where 1=0;") 238 return [description[0] for description in self.cursor.description] 239 240 @_connect 241 def count( 242 self, 243 table: str, 244 match_criteria: list[tuple] | dict | None = None, 245 exact_match: bool = True, 246 ) -> int: 247 """Return number of items in `table`. 248 249 #### :params: 250 251 `match_criteria`: Can be a list of 2-tuples where each 252 tuple is `(columnName, rowValue)` or a dictionary where 253 keys are column names and values are row values. 254 If `None`, all rows from the table will be counted. 255 256 `exact_match`: If `False`, the row value for a given column 257 in `match_criteria` will be matched as a substring. 258 Has no effect if `match_criteria` is `None`. 259 """ 260 query = f"select count(_rowid_) from [{table}]" 261 try: 262 if match_criteria: 263 self.cursor.execute( 264 f"{query} where {self._get_conditions(match_criteria, exact_match)};" 265 ) 266 else: 267 self.cursor.execute(f"{query}") 268 return self.cursor.fetchone()[0] 269 except: 270 return 0 271 272 @_connect 273 def add_row( 274 self, table: str, values: tuple[Any], columns: tuple[str] | None = None 275 ) -> bool: 276 """Add a row of values to a table. 277 278 Returns whether the addition was successful or not. 279 280 #### :params: 281 282 `table`: The table to insert values into. 283 284 `values`: A tuple of values to be inserted into the table. 285 286 `columns`: If `None`, `values` is expected to supply a value for every column in the table. 287 If `columns` is provided, it should contain the same number of elements as `values`.""" 288 parameterizer = ", ".join("?" for _ in values) 289 logger_values = ", ".join(str(value) for value in values) 290 try: 291 if columns: 292 columns_query = ", ".join(column for column in columns) 293 self.cursor.execute( 294 f"insert into [{table}] ({columns_query}) values({parameterizer});", 295 values, 296 ) 297 else: 298 self.cursor.execute( 299 f"insert into [{table}] values({parameterizer});", values 300 ) 301 self.logger.info(f'Added "{logger_values}" to {table} table.') 302 return True 303 except Exception as e: 304 if "constraint" not in str(e).lower(): 305 self.logger.exception( 306 f'Error adding "{logger_values}" to {table} table.' 307 ) 308 else: 309 self.logger.debug(str(e)) 310 return False 311 312 @_connect 313 def add_rows( 314 self, table: str, values: list[tuple[Any]], columns: tuple[str] | None = None 315 ) -> tuple[int, int]: 316 """Add multiple rows of values to a table. 317 318 Returns a tuple containing the number of successful additions and the number of failed additions. 319 320 #### :params: 321 322 `table`: The table to insert values into. 323 324 `values`: A list of tuples of values to be inserted into the table. 325 Each tuple constitutes a single row to be inserted 326 327 `columns`: If `None`, `values` is expected to supply a value for every column in the table. 328 If `columns` is provided, it should contain the same number of elements as `values`.""" 329 successes = 0 330 failures = 0 331 for row in values: 332 if self.add_row(table, row, columns): 333 successes += 1 334 else: 335 failures += 1 336 return (successes, failures) 337 338 @_connect 339 def get_rows( 340 self, 341 table: str, 342 match_criteria: list[tuple] | dict | None = None, 343 exact_match: bool = True, 344 sort_by_column: str | None = None, 345 columns_to_return: list[str] | None = None, 346 return_as_dataframe: bool = False, 347 values_only: bool = False, 348 order_by: str | None = None, 349 limit: str | int | None = None, 350 ) -> list[dict] | list[tuple] | pandas.DataFrame: 351 """Return matching rows from `table`. 352 353 By default, rows will be returned as a list of dictionaries of the form `[{"column_name": value, ...}, ...]` 354 355 356 #### :params: 357 358 `match_criteria`: Can be a list of 2-tuples where each 359 tuple is `(columnName, rowValue)` or a dictionary where 360 keys are column names and values are row values. 361 362 `exact_match`: If `False`, the row value for a given column will be matched as a substring. 363 364 `sort_by_column`: A column name to sort the results by. 365 This will sort results in Python after retrieving them from the db. 366 Use the 'order_by' param to use SQLite engine for ordering. 367 368 `columns_to_return`: Optional list of column names. 369 If provided, the elements returned by this function will only contain the provided columns. 370 Otherwise every column in the row is returned. 371 372 `return_as_dataframe`: Return the results as a `pandas.DataFrame` object. 373 374 `values_only`: Return the results as a list of tuples. 375 376 `order_by`: If given, a `order by {order_by}` clause will be added to the select query. 377 378 `limit`: If given, a `limit {limit}` clause will be added to the select query. 379 """ 380 381 if type(columns_to_return) is str: 382 columns_to_return = [columns_to_return] 383 query = f"select * from [{table}]" 384 matches = [] 385 if match_criteria: 386 query += f" where {self._get_conditions(match_criteria, exact_match)}" 387 if order_by: 388 query += f" order by {order_by}" 389 if limit: 390 query += f" limit {limit}" 391 query += ";" 392 self.cursor.execute(query) 393 matches = self.cursor.fetchall() 394 results = [self._get_dict(table, match, columns_to_return) for match in matches] 395 if sort_by_column: 396 results = sorted(results, key=lambda x: x[sort_by_column]) 397 if return_as_dataframe: 398 return pandas.DataFrame(results) 399 if values_only: 400 return [tuple(row.values()) for row in results] 401 else: 402 return results 403 404 @_connect 405 def find( 406 self, table: str, query_string: str, columns: list[str] | None = None 407 ) -> list[dict]: 408 """Search for rows that contain `query_string` as a substring of any column. 409 410 #### :params: 411 412 `table`: The table to search. 413 414 `query_string`: The substring to search for in all columns. 415 416 `columns`: A list of columns to search for query_string. 417 If None, all columns in the table will be searched. 418 """ 419 if type(columns) is str: 420 columns = [columns] 421 results = [] 422 if not columns: 423 columns = self.get_column_names(table) 424 for column in columns: 425 results.extend( 426 [ 427 row 428 for row in self.get_rows( 429 table, [(column, query_string)], exact_match=False 430 ) 431 if row not in results 432 ] 433 ) 434 return results 435 436 @_connect 437 def delete( 438 self, table: str, match_criteria: list[tuple] | dict, exact_match: bool = True 439 ) -> int: 440 """Delete records from `table`. 441 442 Returns the number of deleted records. 443 444 #### :params: 445 446 `match_criteria`: Can be a list of 2-tuples where each tuple is `(column_name, value)` 447 or a dictionary where keys are column names and values are corresponding values. 448 449 `exact_match`: If `False`, the value for a given column will be matched as a substring. 450 """ 451 conditions = self._get_conditions(match_criteria, exact_match) 452 try: 453 self.cursor.execute(f"delete from [{table}] where {conditions};") 454 num_deletions = self.cursor.rowcount 455 self.logger.info( 456 f'Deleted {num_deletions} rows from "{table}" where {conditions}".' 457 ) 458 return num_deletions 459 except Exception as e: 460 self.logger.debug( 461 f'Error deleting rows from "{table}" where {conditions}.\n{e}' 462 ) 463 return 0 464 465 @_connect 466 def update( 467 self, 468 table: str, 469 column_to_update: str, 470 new_value: Any, 471 match_criteria: list[tuple] | dict | None = None, 472 exact_match: bool = True, 473 ) -> int: 474 """Update the value in `column_to_update` to `new_value` for rows matched with `match_criteria`. 475 476 #### :params: 477 478 `table`: The table to update rows in. 479 480 `column_to_update`: The column to be updated in the matched rows. 481 482 `new_value`: The new value to insert. 483 484 `match_criteria`: Can be a list of 2-tuples where each tuple is `(columnName, rowValue)` 485 or a dictionary where keys are column names and values are corresponding values. 486 If `None`, every row in `table` will be updated. 487 488 `exact_match`: If `False`, `match_criteria` values will be treated as substrings. 489 490 Returns the number of updated rows.""" 491 query = f"update [{table}] set {column_to_update} = ?" 492 conditions = "" 493 if match_criteria: 494 conditions = self._get_conditions(match_criteria, exact_match) 495 query += f" where {conditions}" 496 else: 497 conditions = None 498 query += ";" 499 try: 500 self.cursor.execute( 501 query, 502 (new_value,), 503 ) 504 num_updates = self.cursor.rowcount 505 self.logger.info( 506 f'In {num_updates} rows, updated "{column_to_update}" in "{table}" table to "{new_value}" where {conditions}' 507 ) 508 return num_updates 509 except Exception as e: 510 self.logger.error( 511 f'Failed to update "{column_to_update}" in "{table}" table to "{new_value}" where {conditions}"\n{e}' 512 ) 513 return 0 514 515 @_connect 516 def drop_table(self, table: str) -> bool: 517 """Drop `table` from the database. 518 519 Returns `True` if successful, `False` if not.""" 520 try: 521 self.cursor.execute(f"drop Table [{table}];") 522 self.logger.info(f'Dropped table "{table}"') 523 return True 524 except Exception as e: 525 print(e) 526 self.logger.error(f'Failed to drop table "{table}"') 527 return False 528 529 @_connect 530 def add_column( 531 self, table: str, column: str, _type: str, default_value: str | None = None 532 ): 533 """Add a new column to `table`. 534 535 #### :params: 536 537 `column`: Name of the column to add. 538 539 `_type`: The data type of the new column. 540 541 `default_value`: Optional default value for the column.""" 542 try: 543 if default_value: 544 self.cursor.execute( 545 f"alter table [{table}] add column {column} {_type} default {default_value};" 546 ) 547 self.update(table, column, default_value) 548 else: 549 self.cursor.execute( 550 f"alter table [{table}] add column {column} {_type};" 551 ) 552 self.logger.info(f'Added column "{column}" to "{table}" table.') 553 except Exception as e: 554 self.logger.error(f'Failed to add column "{column}" to "{table}" table.') 555 556 @staticmethod 557 def data_to_string( 558 data: list[dict], sort_key: str | None = None, wrap_to_terminal: bool = True 559 ) -> str: 560 """Uses tabulate to produce pretty string output from a list of dictionaries. 561 562 #### :params: 563 564 `data`: The list of dictionaries to create a grid from. 565 Assumes all dictionaries in list have the same set of keys. 566 567 `sort_key`: Optional dictionary key to sort data with. 568 569 `wrap_to_terminal`: If `True`, the table width will be wrapped to fit within the current terminal window. 570 Pass as `False` if the output is going into something like a `.txt` file.""" 571 return data_to_string(data, sort_key, wrap_to_terminal)
Sqli wrapper so queries don't need to be written except table definitions.
Supports saving and reading dates as datetime objects.
Supports using a context manager.
52 def __init__( 53 self, 54 dbpath: str | Pathier, 55 logger_encoding: str = "utf-8", 56 logger_message_format: str = "{levelname}|-|{asctime}|-|{message}", 57 connection_timeout: float = 10, 58 ): 59 """ 60 #### :params: 61 62 * `dbpath`: String or Path object to database file. 63 If a relative path is given, it will be relative to the 64 current working directory. The log file will be saved to the 65 same directory. 66 67 * `logger_message_format`: `{` style format string for the logger object. 68 69 * `connection_timeout`: The number of seconds to wait when trying to connect to the database before throwing an error.""" 70 self.dbpath = Pathier(dbpath) 71 self.dbname = Pathier(dbpath).name 72 self.dbpath.parent.mkdir(parents=True, exist_ok=True) 73 self._logger_init( 74 encoding=logger_encoding, message_format=logger_message_format 75 ) 76 self.connection_open = False 77 self.connection_timeout = connection_timeout
:params:
dbpath
: String or Path object to database file. If a relative path is given, it will be relative to the current working directory. The log file will be saved to the same directory.logger_message_format
:{
style format string for the logger object.connection_timeout
: The number of seconds to wait when trying to connect to the database before throwing an error.
97 def open(self): 98 """Open connection to db.""" 99 self.connection = sqlite3.connect( 100 self.dbpath, 101 detect_types=sqlite3.PARSE_DECLTYPES | sqlite3.PARSE_COLNAMES, 102 timeout=self.connection_timeout, 103 ) 104 self.connection.execute("pragma foreign_keys = 1;") 105 self.cursor = self.connection.cursor() 106 self.connection_open = True
Open connection to db.
108 def close(self): 109 """Save and close connection to db. 110 111 Call this as soon as you are done using the database if you have 112 multiple threads or processes using the same database.""" 113 if self.connection_open: 114 self.connection.commit() 115 self.connection.close() 116 self.connection_open = False
Save and close connection to db.
Call this as soon as you are done using the database if you have multiple threads or processes using the same database.
189 def vacuum(self): 190 """Reduce disk size of the database with a `VACUUM` query.""" 191 self.query("VACUUM")
Reduce disk size of the database with a VACUUM
query.
193 @_connect 194 def query(self, query_) -> list[Any]: 195 """Execute an arbitrary query and return the results.""" 196 self.cursor.execute(query_) 197 return self.cursor.fetchall()
Execute an arbitrary query and return the results.
199 @_connect 200 def create_tables(self, table_defs: list[str] = []): 201 """Create tables if they don't exist. 202 203 :param `table_defs`: Each definition should be in the form `table_name(column_definitions)`""" 204 if len(table_defs) > 0: 205 table_names = self.get_table_names() 206 for table in table_defs: 207 if table.split("(")[0].strip() not in table_names: 208 self.cursor.execute(f"create table [{table}];") 209 self.logger.info(f'{table.split("(")[0]} table created.')
Create tables if they don't exist.
Parameters
table_defs
: Each definition should be in the formtable_name(column_definitions)
211 @_connect 212 def create_table(self, table: str, column_defs: list[str]): 213 """Create a table if it doesn't exist. 214 215 #### :params: 216 217 `table`: Name of the table to create. 218 219 `column_defs`: List of column definitions in proper Sqlite3 sytax. 220 i.e. `"column_name text unique"` or `"column_name int primary key"` etc.""" 221 if table not in self.get_table_names(): 222 query = f"create table [{table}]({', '.join(column_defs)});" 223 self.cursor.execute(query) 224 self.logger.info(f"'{table}' table created.")
Create a table if it doesn't exist.
:params:
table
: Name of the table to create.
column_defs
: List of column definitions in proper Sqlite3 sytax.
i.e. "column_name text unique"
or "column_name int primary key"
etc.
226 @_connect 227 def get_table_names(self) -> list[str]: 228 """Returns a list of table names from the database.""" 229 self.cursor.execute( 230 'select name from sqlite_Schema where type = "table" and name not like "sqlite_%";' 231 ) 232 return [result[0] for result in self.cursor.fetchall()]
Returns a list of table names from the database.
234 @_connect 235 def get_column_names(self, table: str) -> list[str]: 236 """Return a list of column names from a table.""" 237 self.cursor.execute(f"select * from [{table}] where 1=0;") 238 return [description[0] for description in self.cursor.description]
Return a list of column names from a table.
240 @_connect 241 def count( 242 self, 243 table: str, 244 match_criteria: list[tuple] | dict | None = None, 245 exact_match: bool = True, 246 ) -> int: 247 """Return number of items in `table`. 248 249 #### :params: 250 251 `match_criteria`: Can be a list of 2-tuples where each 252 tuple is `(columnName, rowValue)` or a dictionary where 253 keys are column names and values are row values. 254 If `None`, all rows from the table will be counted. 255 256 `exact_match`: If `False`, the row value for a given column 257 in `match_criteria` will be matched as a substring. 258 Has no effect if `match_criteria` is `None`. 259 """ 260 query = f"select count(_rowid_) from [{table}]" 261 try: 262 if match_criteria: 263 self.cursor.execute( 264 f"{query} where {self._get_conditions(match_criteria, exact_match)};" 265 ) 266 else: 267 self.cursor.execute(f"{query}") 268 return self.cursor.fetchone()[0] 269 except: 270 return 0
Return number of items in table
.
:params:
match_criteria
: Can be a list of 2-tuples where each
tuple is (columnName, rowValue)
or a dictionary where
keys are column names and values are row values.
If None
, all rows from the table will be counted.
exact_match
: If False
, the row value for a given column
in match_criteria
will be matched as a substring.
Has no effect if match_criteria
is None
.
272 @_connect 273 def add_row( 274 self, table: str, values: tuple[Any], columns: tuple[str] | None = None 275 ) -> bool: 276 """Add a row of values to a table. 277 278 Returns whether the addition was successful or not. 279 280 #### :params: 281 282 `table`: The table to insert values into. 283 284 `values`: A tuple of values to be inserted into the table. 285 286 `columns`: If `None`, `values` is expected to supply a value for every column in the table. 287 If `columns` is provided, it should contain the same number of elements as `values`.""" 288 parameterizer = ", ".join("?" for _ in values) 289 logger_values = ", ".join(str(value) for value in values) 290 try: 291 if columns: 292 columns_query = ", ".join(column for column in columns) 293 self.cursor.execute( 294 f"insert into [{table}] ({columns_query}) values({parameterizer});", 295 values, 296 ) 297 else: 298 self.cursor.execute( 299 f"insert into [{table}] values({parameterizer});", values 300 ) 301 self.logger.info(f'Added "{logger_values}" to {table} table.') 302 return True 303 except Exception as e: 304 if "constraint" not in str(e).lower(): 305 self.logger.exception( 306 f'Error adding "{logger_values}" to {table} table.' 307 ) 308 else: 309 self.logger.debug(str(e)) 310 return False
Add a row of values to a table.
Returns whether the addition was successful or not.
:params:
table
: The table to insert values into.
values
: A tuple of values to be inserted into the table.
columns
: If None
, values
is expected to supply a value for every column in the table.
If columns
is provided, it should contain the same number of elements as values
.
312 @_connect 313 def add_rows( 314 self, table: str, values: list[tuple[Any]], columns: tuple[str] | None = None 315 ) -> tuple[int, int]: 316 """Add multiple rows of values to a table. 317 318 Returns a tuple containing the number of successful additions and the number of failed additions. 319 320 #### :params: 321 322 `table`: The table to insert values into. 323 324 `values`: A list of tuples of values to be inserted into the table. 325 Each tuple constitutes a single row to be inserted 326 327 `columns`: If `None`, `values` is expected to supply a value for every column in the table. 328 If `columns` is provided, it should contain the same number of elements as `values`.""" 329 successes = 0 330 failures = 0 331 for row in values: 332 if self.add_row(table, row, columns): 333 successes += 1 334 else: 335 failures += 1 336 return (successes, failures)
Add multiple rows of values to a table.
Returns a tuple containing the number of successful additions and the number of failed additions.
:params:
table
: The table to insert values into.
values
: A list of tuples of values to be inserted into the table.
Each tuple constitutes a single row to be inserted
columns
: If None
, values
is expected to supply a value for every column in the table.
If columns
is provided, it should contain the same number of elements as values
.
338 @_connect 339 def get_rows( 340 self, 341 table: str, 342 match_criteria: list[tuple] | dict | None = None, 343 exact_match: bool = True, 344 sort_by_column: str | None = None, 345 columns_to_return: list[str] | None = None, 346 return_as_dataframe: bool = False, 347 values_only: bool = False, 348 order_by: str | None = None, 349 limit: str | int | None = None, 350 ) -> list[dict] | list[tuple] | pandas.DataFrame: 351 """Return matching rows from `table`. 352 353 By default, rows will be returned as a list of dictionaries of the form `[{"column_name": value, ...}, ...]` 354 355 356 #### :params: 357 358 `match_criteria`: Can be a list of 2-tuples where each 359 tuple is `(columnName, rowValue)` or a dictionary where 360 keys are column names and values are row values. 361 362 `exact_match`: If `False`, the row value for a given column will be matched as a substring. 363 364 `sort_by_column`: A column name to sort the results by. 365 This will sort results in Python after retrieving them from the db. 366 Use the 'order_by' param to use SQLite engine for ordering. 367 368 `columns_to_return`: Optional list of column names. 369 If provided, the elements returned by this function will only contain the provided columns. 370 Otherwise every column in the row is returned. 371 372 `return_as_dataframe`: Return the results as a `pandas.DataFrame` object. 373 374 `values_only`: Return the results as a list of tuples. 375 376 `order_by`: If given, a `order by {order_by}` clause will be added to the select query. 377 378 `limit`: If given, a `limit {limit}` clause will be added to the select query. 379 """ 380 381 if type(columns_to_return) is str: 382 columns_to_return = [columns_to_return] 383 query = f"select * from [{table}]" 384 matches = [] 385 if match_criteria: 386 query += f" where {self._get_conditions(match_criteria, exact_match)}" 387 if order_by: 388 query += f" order by {order_by}" 389 if limit: 390 query += f" limit {limit}" 391 query += ";" 392 self.cursor.execute(query) 393 matches = self.cursor.fetchall() 394 results = [self._get_dict(table, match, columns_to_return) for match in matches] 395 if sort_by_column: 396 results = sorted(results, key=lambda x: x[sort_by_column]) 397 if return_as_dataframe: 398 return pandas.DataFrame(results) 399 if values_only: 400 return [tuple(row.values()) for row in results] 401 else: 402 return results
Return matching rows from table
.
By default, rows will be returned as a list of dictionaries of the form [{"column_name": value, ...}, ...]
:params:
match_criteria
: Can be a list of 2-tuples where each
tuple is (columnName, rowValue)
or a dictionary where
keys are column names and values are row values.
exact_match
: If False
, the row value for a given column will be matched as a substring.
sort_by_column
: A column name to sort the results by.
This will sort results in Python after retrieving them from the db.
Use the 'order_by' param to use SQLite engine for ordering.
columns_to_return
: Optional list of column names.
If provided, the elements returned by this function will only contain the provided columns.
Otherwise every column in the row is returned.
return_as_dataframe
: Return the results as a pandas.DataFrame
object.
values_only
: Return the results as a list of tuples.
order_by
: If given, a order by {order_by}
clause will be added to the select query.
limit
: If given, a limit {limit}
clause will be added to the select query.
404 @_connect 405 def find( 406 self, table: str, query_string: str, columns: list[str] | None = None 407 ) -> list[dict]: 408 """Search for rows that contain `query_string` as a substring of any column. 409 410 #### :params: 411 412 `table`: The table to search. 413 414 `query_string`: The substring to search for in all columns. 415 416 `columns`: A list of columns to search for query_string. 417 If None, all columns in the table will be searched. 418 """ 419 if type(columns) is str: 420 columns = [columns] 421 results = [] 422 if not columns: 423 columns = self.get_column_names(table) 424 for column in columns: 425 results.extend( 426 [ 427 row 428 for row in self.get_rows( 429 table, [(column, query_string)], exact_match=False 430 ) 431 if row not in results 432 ] 433 ) 434 return results
Search for rows that contain query_string
as a substring of any column.
:params:
table
: The table to search.
query_string
: The substring to search for in all columns.
columns
: A list of columns to search for query_string.
If None, all columns in the table will be searched.
436 @_connect 437 def delete( 438 self, table: str, match_criteria: list[tuple] | dict, exact_match: bool = True 439 ) -> int: 440 """Delete records from `table`. 441 442 Returns the number of deleted records. 443 444 #### :params: 445 446 `match_criteria`: Can be a list of 2-tuples where each tuple is `(column_name, value)` 447 or a dictionary where keys are column names and values are corresponding values. 448 449 `exact_match`: If `False`, the value for a given column will be matched as a substring. 450 """ 451 conditions = self._get_conditions(match_criteria, exact_match) 452 try: 453 self.cursor.execute(f"delete from [{table}] where {conditions};") 454 num_deletions = self.cursor.rowcount 455 self.logger.info( 456 f'Deleted {num_deletions} rows from "{table}" where {conditions}".' 457 ) 458 return num_deletions 459 except Exception as e: 460 self.logger.debug( 461 f'Error deleting rows from "{table}" where {conditions}.\n{e}' 462 ) 463 return 0
Delete records from table
.
Returns the number of deleted records.
:params:
match_criteria
: Can be a list of 2-tuples where each tuple is (column_name, value)
or a dictionary where keys are column names and values are corresponding values.
exact_match
: If False
, the value for a given column will be matched as a substring.
465 @_connect 466 def update( 467 self, 468 table: str, 469 column_to_update: str, 470 new_value: Any, 471 match_criteria: list[tuple] | dict | None = None, 472 exact_match: bool = True, 473 ) -> int: 474 """Update the value in `column_to_update` to `new_value` for rows matched with `match_criteria`. 475 476 #### :params: 477 478 `table`: The table to update rows in. 479 480 `column_to_update`: The column to be updated in the matched rows. 481 482 `new_value`: The new value to insert. 483 484 `match_criteria`: Can be a list of 2-tuples where each tuple is `(columnName, rowValue)` 485 or a dictionary where keys are column names and values are corresponding values. 486 If `None`, every row in `table` will be updated. 487 488 `exact_match`: If `False`, `match_criteria` values will be treated as substrings. 489 490 Returns the number of updated rows.""" 491 query = f"update [{table}] set {column_to_update} = ?" 492 conditions = "" 493 if match_criteria: 494 conditions = self._get_conditions(match_criteria, exact_match) 495 query += f" where {conditions}" 496 else: 497 conditions = None 498 query += ";" 499 try: 500 self.cursor.execute( 501 query, 502 (new_value,), 503 ) 504 num_updates = self.cursor.rowcount 505 self.logger.info( 506 f'In {num_updates} rows, updated "{column_to_update}" in "{table}" table to "{new_value}" where {conditions}' 507 ) 508 return num_updates 509 except Exception as e: 510 self.logger.error( 511 f'Failed to update "{column_to_update}" in "{table}" table to "{new_value}" where {conditions}"\n{e}' 512 ) 513 return 0
Update the value in column_to_update
to new_value
for rows matched with match_criteria
.
:params:
table
: The table to update rows in.
column_to_update
: The column to be updated in the matched rows.
new_value
: The new value to insert.
match_criteria
: Can be a list of 2-tuples where each tuple is (columnName, rowValue)
or a dictionary where keys are column names and values are corresponding values.
If None
, every row in table
will be updated.
exact_match
: If False
, match_criteria
values will be treated as substrings.
Returns the number of updated rows.
515 @_connect 516 def drop_table(self, table: str) -> bool: 517 """Drop `table` from the database. 518 519 Returns `True` if successful, `False` if not.""" 520 try: 521 self.cursor.execute(f"drop Table [{table}];") 522 self.logger.info(f'Dropped table "{table}"') 523 return True 524 except Exception as e: 525 print(e) 526 self.logger.error(f'Failed to drop table "{table}"') 527 return False
Drop table
from the database.
Returns True
if successful, False
if not.
529 @_connect 530 def add_column( 531 self, table: str, column: str, _type: str, default_value: str | None = None 532 ): 533 """Add a new column to `table`. 534 535 #### :params: 536 537 `column`: Name of the column to add. 538 539 `_type`: The data type of the new column. 540 541 `default_value`: Optional default value for the column.""" 542 try: 543 if default_value: 544 self.cursor.execute( 545 f"alter table [{table}] add column {column} {_type} default {default_value};" 546 ) 547 self.update(table, column, default_value) 548 else: 549 self.cursor.execute( 550 f"alter table [{table}] add column {column} {_type};" 551 ) 552 self.logger.info(f'Added column "{column}" to "{table}" table.') 553 except Exception as e: 554 self.logger.error(f'Failed to add column "{column}" to "{table}" table.')
Add a new column to table
.
:params:
column
: Name of the column to add.
_type
: The data type of the new column.
default_value
: Optional default value for the column.
556 @staticmethod 557 def data_to_string( 558 data: list[dict], sort_key: str | None = None, wrap_to_terminal: bool = True 559 ) -> str: 560 """Uses tabulate to produce pretty string output from a list of dictionaries. 561 562 #### :params: 563 564 `data`: The list of dictionaries to create a grid from. 565 Assumes all dictionaries in list have the same set of keys. 566 567 `sort_key`: Optional dictionary key to sort data with. 568 569 `wrap_to_terminal`: If `True`, the table width will be wrapped to fit within the current terminal window. 570 Pass as `False` if the output is going into something like a `.txt` file.""" 571 return data_to_string(data, sort_key, wrap_to_terminal)
Uses tabulate to produce pretty string output from a list of dictionaries.
:params:
data
: The list of dictionaries to create a grid from.
Assumes all dictionaries in list have the same set of keys.
sort_key
: Optional dictionary key to sort data with.
wrap_to_terminal
: If True
, the table width will be wrapped to fit within the current terminal window.
Pass as False
if the output is going into something like a .txt
file.
574def data_to_string( 575 data: list[dict], sort_key: str | None = None, wrap_to_terminal: bool = True 576) -> str: 577 """Uses tabulate to produce pretty string output from a list of dictionaries. 578 579 #### :params: 580 581 `data`: The list of dictionaries to create a grid from. 582 Assumes all dictionaries in list have the same set of keys. 583 584 `sort_key`: Optional dictionary key to sort data with. 585 586 `wrap_to_terminal`: If `True`, the table width will be wrapped to fit within the current terminal window. 587 Pass as `False` if the output is going into something like a `.txt` file.""" 588 if len(data) == 0: 589 return "" 590 if sort_key: 591 data = sorted(data, key=lambda d: d[sort_key]) 592 for i, d in enumerate(data): 593 for k in d: 594 data[i][k] = str(data[i][k]) 595 596 too_wide = True 597 terminal_width = os.get_terminal_size().columns 598 max_col_widths = terminal_width 599 # Make an output with effectively unrestricted column widths 600 # to see if shrinking is necessary 601 output = tabulate( 602 data, 603 headers="keys", 604 disable_numparse=True, 605 tablefmt="grid", 606 maxcolwidths=max_col_widths, 607 ) 608 current_width = output.index("\n") 609 if current_width < terminal_width: 610 too_wide = False 611 if wrap_to_terminal and too_wide: 612 print("Resizing grid to fit within the terminal...\n") 613 previous_col_widths = max_col_widths 614 acceptable_width = terminal_width - 10 615 while too_wide and max_col_widths > 1: 616 if current_width >= terminal_width: 617 previous_col_widths = max_col_widths 618 max_col_widths = int(max_col_widths * 0.5) 619 elif current_width < terminal_width: 620 # Without lowering acceptable_width, this condition will cause infinite loop 621 if max_col_widths == previous_col_widths - 1: 622 acceptable_width -= 10 623 max_col_widths = int( 624 max_col_widths + (0.5 * (previous_col_widths - max_col_widths)) 625 ) 626 output = tabulate( 627 data, 628 headers="keys", 629 disable_numparse=True, 630 tablefmt="grid", 631 maxcolwidths=max_col_widths, 632 ) 633 current_width = output.index("\n") 634 if acceptable_width < current_width < terminal_width: 635 too_wide = False 636 if too_wide: 637 print("Couldn't resize grid to fit within the terminal.") 638 return str(data) 639 return output
Uses tabulate to produce pretty string output from a list of dictionaries.
:params:
data
: The list of dictionaries to create a grid from.
Assumes all dictionaries in list have the same set of keys.
sort_key
: Optional dictionary key to sort data with.
wrap_to_terminal
: If True
, the table width will be wrapped to fit within the current terminal window.
Pass as False
if the output is going into something like a .txt
file.