databased.databased
1import logging 2import os 3import sqlite3 4from datetime import datetime 5from functools import wraps 6from typing import Any 7 8import pandas 9from pathier import Pathier 10from tabulate import tabulate 11 12 13def _connect(func): 14 """Decorator to open db connection if it isn't already open.""" 15 16 @wraps(func) 17 def inner(self, *args, **kwargs): 18 if not self.connection_open: 19 self.open() 20 results = func(self, *args, **kwargs) 21 return results 22 23 return inner 24 25 26class DataBased: 27 """Sqli wrapper so queries don't need to be written except table definitions. 28 29 Supports saving and reading dates as datetime objects. 30 31 Supports using a context manager.""" 32 33 def __init__( 34 self, 35 dbpath: str | Pathier, 36 logger_encoding: str = "utf-8", 37 logger_message_format: str = "{levelname}|-|{asctime}|-|{message}", 38 ): 39 """ 40 #### :params: 41 42 `dbpath`: String or Path object to database file. 43 If a relative path is given, it will be relative to the 44 current working directory. The log file will be saved to the 45 same directory. 46 47 `logger_message_format`: '{' style format string for the logger object.""" 48 self.dbpath = Pathier(dbpath) 49 self.dbname = Pathier(dbpath).name 50 self.dbpath.parent.mkdir(parents=True, exist_ok=True) 51 self._logger_init( 52 encoding=logger_encoding, message_format=logger_message_format 53 ) 54 self.connection_open = False 55 56 def __enter__(self): 57 self.open() 58 return self 59 60 def __exit__(self, exception_type, exception_value, exception_traceback): 61 self.close() 62 63 def open(self): 64 """Open connection to db.""" 65 self.connection = sqlite3.connect( 66 self.dbpath, 67 detect_types=sqlite3.PARSE_DECLTYPES | sqlite3.PARSE_COLNAMES, 68 timeout=10, 69 ) 70 self.connection.execute("pragma foreign_keys = 1;") 71 self.cursor = self.connection.cursor() 72 self.connection_open = True 73 74 def close(self): 75 """Save and close connection to db. 76 77 Call this as soon as you are done using the database if you have 78 multiple threads or processes using the same database.""" 79 if self.connection_open: 80 self.connection.commit() 81 self.connection.close() 82 self.connection_open = False 83 84 def _logger_init( 85 self, 86 message_format: str = "{levelname}|-|{asctime}|-|{message}", 87 encoding: str = "utf-8", 88 ): 89 """:param `message_format`: '{' style format string""" 90 self.logger = logging.getLogger(self.dbname) 91 if not self.logger.hasHandlers(): 92 handler = logging.FileHandler( 93 str(self.dbpath).replace(".", "") + ".log", encoding=encoding 94 ) 95 handler.setFormatter( 96 logging.Formatter( 97 message_format, style="{", datefmt="%m/%d/%Y %I:%M:%S %p" 98 ) 99 ) 100 self.logger.addHandler(handler) 101 self.logger.setLevel(logging.INFO) 102 103 def _get_dict( 104 self, table: str, values: list, columns_to_return: list[str] | None = None 105 ) -> dict: 106 """Converts the values of a row into a dictionary with column names as keys. 107 108 #### :params: 109 110 `table`: The table that values were pulled from. 111 112 `values`: List of values expected to be the same quantity 113 and in the same order as the column names of table. 114 115 `columns_to_return`: An optional list of column names. 116 If given, only these columns will be included in the returned dictionary. 117 Otherwise all columns and values are returned.""" 118 return { 119 column: value 120 for column, value in zip(self.get_column_names(table), values) 121 if not columns_to_return or column in columns_to_return 122 } 123 124 def _get_conditions( 125 self, match_criteria: list[tuple] | dict, exact_match: bool = True 126 ) -> str: 127 """Builds and returns the conditional portion of a query. 128 129 #### :params: 130 131 `match_criteria`: Can be a list of 2-tuples where each 132 tuple is `(columnName, rowValue)` or a dictionary where 133 keys are column names and values are row values. 134 135 `exact_match`: If `False`, the row value for a given column 136 will be matched as a substring. 137 138 Usage e.g.: 139 140 >>> self.cursor.execute(f'select * from {table} where {conditions};')""" 141 if type(match_criteria) == dict: 142 match_criteria = [(k, v) for k, v in match_criteria.items()] 143 if exact_match: 144 conditions = " and ".join( 145 f'"{column_row[0]}" = "{column_row[1]}"' 146 for column_row in match_criteria 147 ) 148 else: 149 conditions = " and ".join( 150 f'"{column_row[0]}" like "%{column_row[1]}%"' 151 for column_row in match_criteria 152 ) 153 return f"({conditions})" 154 155 @_connect 156 def query(self, query_) -> list[Any]: 157 """Execute an arbitrary query and return the results.""" 158 self.cursor.execute(query_) 159 return self.cursor.fetchall() 160 161 @_connect 162 def create_tables(self, table_defs: list[str] = []): 163 """Create tables if they don't exist. 164 165 :param `table_defs`: Each definition should be in the form `table_name(column_definitions)`""" 166 if len(table_defs) > 0: 167 table_names = self.get_table_names() 168 for table in table_defs: 169 if table.split("(")[0].strip() not in table_names: 170 self.cursor.execute(f"create table {table};") 171 self.logger.info(f'{table.split("(")[0]} table created.') 172 173 @_connect 174 def create_table(self, table: str, column_defs: list[str]): 175 """Create a table if it doesn't exist. 176 177 #### :params: 178 179 `table`: Name of the table to create. 180 181 `column_defs`: List of column definitions in proper Sqlite3 sytax. 182 i.e. `"column_name text unique"` or `"column_name int primary key"` etc.""" 183 if table not in self.get_table_names(): 184 query = f"create table {table}({', '.join(column_defs)});" 185 self.cursor.execute(query) 186 self.logger.info(f"'{table}' table created.") 187 188 @_connect 189 def get_table_names(self) -> list[str]: 190 """Returns a list of table names from the database.""" 191 self.cursor.execute( 192 'select name from sqlite_Schema where type = "table" and name not like "sqlite_%";' 193 ) 194 return [result[0] for result in self.cursor.fetchall()] 195 196 @_connect 197 def get_column_names(self, table: str) -> list[str]: 198 """Return a list of column names from a table.""" 199 self.cursor.execute(f"select * from {table} where 1=0") 200 return [description[0] for description in self.cursor.description] 201 202 @_connect 203 def count( 204 self, 205 table: str, 206 match_criteria: list[tuple] | dict | None = None, 207 exact_match: bool = True, 208 ) -> int: 209 """Return number of items in `table`. 210 211 #### :params: 212 213 `match_criteria`: Can be a list of 2-tuples where each 214 tuple is `(columnName, rowValue)` or a dictionary where 215 keys are column names and values are row values. 216 If `None`, all rows from the table will be counted. 217 218 `exact_match`: If `False`, the row value for a given column 219 in `match_criteria` will be matched as a substring. 220 Has no effect if `match_criteria` is `None`. 221 """ 222 query = f"select count(_rowid_) from {table}" 223 try: 224 if match_criteria: 225 self.cursor.execute( 226 f"{query} where {self._get_conditions(match_criteria, exact_match)};" 227 ) 228 else: 229 self.cursor.execute(f"{query}") 230 return self.cursor.fetchone()[0] 231 except: 232 return 0 233 234 @_connect 235 def add_row( 236 self, table: str, values: tuple[Any], columns: tuple[str] | None = None 237 ): 238 """Add a row of values to a table. 239 240 #### :params: 241 242 `table`: The table to insert values into. 243 244 `values`: A tuple of values to be inserted into the table. 245 246 `columns`: If `None`, `values` is expected to supply a value for every column in the table. 247 If `columns` is provided, it should contain the same number of elements as `values`.""" 248 parameterizer = ", ".join("?" for _ in values) 249 logger_values = ", ".join(str(value) for value in values) 250 try: 251 if columns: 252 columns_query = ", ".join(column for column in columns) 253 self.cursor.execute( 254 f"insert into {table} ({columns_query}) values({parameterizer});", 255 values, 256 ) 257 else: 258 self.cursor.execute( 259 f"insert into {table} values({parameterizer});", values 260 ) 261 self.logger.info(f'Added "{logger_values}" to {table} table.') 262 except Exception as e: 263 if "constraint" not in str(e).lower(): 264 self.logger.exception( 265 f'Error adding "{logger_values}" to {table} table.' 266 ) 267 else: 268 self.logger.debug(str(e)) 269 270 @_connect 271 def get_rows( 272 self, 273 table: str, 274 match_criteria: list[tuple] | dict | None = None, 275 exact_match: bool = True, 276 sort_by_column: str | None = None, 277 columns_to_return: list[str] | None = None, 278 return_as_dataframe: bool = False, 279 values_only: bool = False, 280 order_by: str | None = None, 281 limit: str | int | None = None, 282 ) -> list[dict] | list[tuple] | pandas.DataFrame: 283 """Return matching rows from `table`. 284 285 By default, rows will be returned as a list of dictionaries of the form `[{"column_name": value, ...}, ...]` 286 287 288 #### :params: 289 290 `match_criteria`: Can be a list of 2-tuples where each 291 tuple is `(columnName, rowValue)` or a dictionary where 292 keys are column names and values are row values. 293 294 `exact_match`: If `False`, the row value for a given column will be matched as a substring. 295 296 `sort_by_column`: A column name to sort the results by. 297 This will sort results in Python after retrieving them from the db. 298 Use the 'order_by' param to use SQLite engine for ordering. 299 300 `columns_to_return`: Optional list of column names. 301 If provided, the elements returned by this function will only contain the provided columns. 302 Otherwise every column in the row is returned. 303 304 `return_as_dataframe`: Return the results as a `pandas.DataFrame` object. 305 306 `values_only`: Return the results as a list of tuples. 307 308 `order_by`: If given, a `order by {order_by}` clause will be added to the select query. 309 310 `limit`: If given, a `limit {limit}` clause will be added to the select query. 311 """ 312 313 if type(columns_to_return) is str: 314 columns_to_return = [columns_to_return] 315 query = f"select * from {table}" 316 matches = [] 317 if match_criteria: 318 query += f" where {self._get_conditions(match_criteria, exact_match)}" 319 if order_by: 320 query += f" order by {order_by}" 321 if limit: 322 query += f" limit {limit}" 323 query += ";" 324 self.cursor.execute(query) 325 matches = self.cursor.fetchall() 326 results = [self._get_dict(table, match, columns_to_return) for match in matches] 327 if sort_by_column: 328 results = sorted(results, key=lambda x: x[sort_by_column]) 329 if return_as_dataframe: 330 return pandas.DataFrame(results) 331 if values_only: 332 return [tuple(row.values()) for row in results] 333 else: 334 return results 335 336 @_connect 337 def find( 338 self, table: str, query_string: str, columns: list[str] | None = None 339 ) -> list[dict]: 340 """Search for rows that contain `query_string` as a substring of any column. 341 342 #### :params: 343 344 `table`: The table to search. 345 346 `query_string`: The substring to search for in all columns. 347 348 `columns`: A list of columns to search for query_string. 349 If None, all columns in the table will be searched. 350 """ 351 if type(columns) is str: 352 columns = [columns] 353 results = [] 354 if not columns: 355 columns = self.get_column_names(table) 356 for column in columns: 357 results.extend( 358 [ 359 row 360 for row in self.get_rows( 361 table, [(column, query_string)], exact_match=False 362 ) 363 if row not in results 364 ] 365 ) 366 return results 367 368 @_connect 369 def delete( 370 self, table: str, match_criteria: list[tuple] | dict, exact_match: bool = True 371 ) -> int: 372 """Delete records from `table`. 373 374 Returns the number of deleted records. 375 376 #### :params: 377 378 `match_criteria`: Can be a list of 2-tuples where each tuple is `(column_name, value)` 379 or a dictionary where keys are column names and values are corresponding values. 380 381 `exact_match`: If `False`, the value for a given column will be matched as a substring. 382 """ 383 num_matches = self.count(table, match_criteria, exact_match) 384 conditions = self._get_conditions(match_criteria, exact_match) 385 try: 386 self.cursor.execute(f"delete from {table} where {conditions};") 387 self.logger.info( 388 f'Deleted {num_matches} from "{table}" where {conditions}".' 389 ) 390 return num_matches 391 except Exception as e: 392 self.logger.debug(f'Error deleting from "{table}" where {conditions}.\n{e}') 393 return 0 394 395 @_connect 396 def update( 397 self, 398 table: str, 399 column_to_update: str, 400 new_value: Any, 401 match_criteria: list[tuple] | dict | None = None, 402 ) -> bool: 403 """Update the value in `column_to_update` to `new_value` for rows matched with `match_criteria`. 404 405 #### :params: 406 407 `table`: The table to update rows in. 408 409 `column_to_update`: The column to be updated in the matched rows. 410 411 `new_value`: The new value to insert. 412 413 `match_criteria`: Can be a list of 2-tuples where each tuple is `(columnName, rowValue)` 414 or a dictionary where keys are column names and values are corresponding values. 415 If `None`, every row in `table` will be updated. 416 417 Returns `True` if successful, `False` if not.""" 418 query = f"update {table} set {column_to_update} = ?" 419 conditions = "" 420 if match_criteria: 421 if self.count(table, match_criteria) == 0: 422 self.logger.info( 423 f"Couldn't find matching records in {table} table to update to '{new_value}'" 424 ) 425 return False 426 conditions = self._get_conditions(match_criteria) 427 query += f" where {conditions}" 428 else: 429 conditions = None 430 query += ";" 431 try: 432 self.cursor.execute( 433 query, 434 (new_value,), 435 ) 436 self.logger.info( 437 f'Updated "{column_to_update}" in "{table}" table to "{new_value}" where {conditions}' 438 ) 439 return True 440 except Exception as e: 441 self.logger.error( 442 f'Failed to update "{column_to_update}" in "{table}" table to "{new_value}" where {conditions}"\n{e}' 443 ) 444 return False 445 446 @_connect 447 def drop_table(self, table: str) -> bool: 448 """Drop `table` from the database. 449 450 Returns `True` if successful, `False` if not.""" 451 try: 452 self.cursor.execute(f"drop Table {table};") 453 self.logger.info(f'Dropped table "{table}"') 454 return True 455 except Exception as e: 456 print(e) 457 self.logger.error(f'Failed to drop table "{table}"') 458 return False 459 460 @_connect 461 def add_column( 462 self, table: str, column: str, _type: str, default_value: str | None = None 463 ): 464 """Add a new column to `table`. 465 466 #### :params: 467 468 `column`: Name of the column to add. 469 470 `_type`: The data type of the new column. 471 472 `default_value`: Optional default value for the column.""" 473 try: 474 if default_value: 475 self.cursor.execute( 476 f"alter table {table} add column {column} {_type} default {default_value};" 477 ) 478 self.update(table, column, default_value) 479 else: 480 self.cursor.execute(f"alter table {table} add column {column} {_type};") 481 self.logger.info(f'Added column "{column}" to "{table}" table.') 482 except Exception as e: 483 self.logger.error(f'Failed to add column "{column}" to "{table}" table.') 484 485 @staticmethod 486 def data_to_string( 487 data: list[dict], sort_key: str | None = None, wrap_to_terminal: bool = True 488 ) -> str: 489 """Uses tabulate to produce pretty string output from a list of dictionaries. 490 491 #### :params: 492 493 `data`: The list of dictionaries to create a grid from. 494 Assumes all dictionaries in list have the same set of keys. 495 496 `sort_key`: Optional dictionary key to sort data with. 497 498 `wrap_to_terminal`: If `True`, the table width will be wrapped to fit within the current terminal window. 499 Pass as `False` if the output is going into something like a `.txt` file.""" 500 return data_to_string(data, sort_key, wrap_to_terminal) 501 502 503def data_to_string( 504 data: list[dict], sort_key: str | None = None, wrap_to_terminal: bool = True 505) -> str: 506 """Uses tabulate to produce pretty string output from a list of dictionaries. 507 508 #### :params: 509 510 `data`: The list of dictionaries to create a grid from. 511 Assumes all dictionaries in list have the same set of keys. 512 513 `sort_key`: Optional dictionary key to sort data with. 514 515 `wrap_to_terminal`: If `True`, the table width will be wrapped to fit within the current terminal window. 516 Pass as `False` if the output is going into something like a `.txt` file.""" 517 if len(data) == 0: 518 return "" 519 if sort_key: 520 data = sorted(data, key=lambda d: d[sort_key]) 521 for i, d in enumerate(data): 522 for k in d: 523 data[i][k] = str(data[i][k]) 524 525 too_wide = True 526 terminal_width = os.get_terminal_size().columns 527 max_col_widths = terminal_width 528 # Make an output with effectively unrestricted column widths 529 # to see if shrinking is necessary 530 output = tabulate( 531 data, 532 headers="keys", 533 disable_numparse=True, 534 tablefmt="grid", 535 maxcolwidths=max_col_widths, 536 ) 537 current_width = output.index("\n") 538 if current_width < terminal_width: 539 too_wide = False 540 if wrap_to_terminal and too_wide: 541 print("Resizing grid to fit within the terminal...\n") 542 previous_col_widths = max_col_widths 543 acceptable_width = terminal_width - 10 544 while too_wide and max_col_widths > 1: 545 if current_width >= terminal_width: 546 previous_col_widths = max_col_widths 547 max_col_widths = int(max_col_widths * 0.5) 548 elif current_width < terminal_width: 549 # Without lowering acceptable_width, this condition will cause infinite loop 550 if max_col_widths == previous_col_widths - 1: 551 acceptable_width -= 10 552 max_col_widths = int( 553 max_col_widths + (0.5 * (previous_col_widths - max_col_widths)) 554 ) 555 output = tabulate( 556 data, 557 headers="keys", 558 disable_numparse=True, 559 tablefmt="grid", 560 maxcolwidths=max_col_widths, 561 ) 562 current_width = output.index("\n") 563 if acceptable_width < current_width < terminal_width: 564 too_wide = False 565 if too_wide: 566 print("Couldn't resize grid to fit within the terminal.") 567 return str(data) 568 return output
27class DataBased: 28 """Sqli wrapper so queries don't need to be written except table definitions. 29 30 Supports saving and reading dates as datetime objects. 31 32 Supports using a context manager.""" 33 34 def __init__( 35 self, 36 dbpath: str | Pathier, 37 logger_encoding: str = "utf-8", 38 logger_message_format: str = "{levelname}|-|{asctime}|-|{message}", 39 ): 40 """ 41 #### :params: 42 43 `dbpath`: String or Path object to database file. 44 If a relative path is given, it will be relative to the 45 current working directory. The log file will be saved to the 46 same directory. 47 48 `logger_message_format`: '{' style format string for the logger object.""" 49 self.dbpath = Pathier(dbpath) 50 self.dbname = Pathier(dbpath).name 51 self.dbpath.parent.mkdir(parents=True, exist_ok=True) 52 self._logger_init( 53 encoding=logger_encoding, message_format=logger_message_format 54 ) 55 self.connection_open = False 56 57 def __enter__(self): 58 self.open() 59 return self 60 61 def __exit__(self, exception_type, exception_value, exception_traceback): 62 self.close() 63 64 def open(self): 65 """Open connection to db.""" 66 self.connection = sqlite3.connect( 67 self.dbpath, 68 detect_types=sqlite3.PARSE_DECLTYPES | sqlite3.PARSE_COLNAMES, 69 timeout=10, 70 ) 71 self.connection.execute("pragma foreign_keys = 1;") 72 self.cursor = self.connection.cursor() 73 self.connection_open = True 74 75 def close(self): 76 """Save and close connection to db. 77 78 Call this as soon as you are done using the database if you have 79 multiple threads or processes using the same database.""" 80 if self.connection_open: 81 self.connection.commit() 82 self.connection.close() 83 self.connection_open = False 84 85 def _logger_init( 86 self, 87 message_format: str = "{levelname}|-|{asctime}|-|{message}", 88 encoding: str = "utf-8", 89 ): 90 """:param `message_format`: '{' style format string""" 91 self.logger = logging.getLogger(self.dbname) 92 if not self.logger.hasHandlers(): 93 handler = logging.FileHandler( 94 str(self.dbpath).replace(".", "") + ".log", encoding=encoding 95 ) 96 handler.setFormatter( 97 logging.Formatter( 98 message_format, style="{", datefmt="%m/%d/%Y %I:%M:%S %p" 99 ) 100 ) 101 self.logger.addHandler(handler) 102 self.logger.setLevel(logging.INFO) 103 104 def _get_dict( 105 self, table: str, values: list, columns_to_return: list[str] | None = None 106 ) -> dict: 107 """Converts the values of a row into a dictionary with column names as keys. 108 109 #### :params: 110 111 `table`: The table that values were pulled from. 112 113 `values`: List of values expected to be the same quantity 114 and in the same order as the column names of table. 115 116 `columns_to_return`: An optional list of column names. 117 If given, only these columns will be included in the returned dictionary. 118 Otherwise all columns and values are returned.""" 119 return { 120 column: value 121 for column, value in zip(self.get_column_names(table), values) 122 if not columns_to_return or column in columns_to_return 123 } 124 125 def _get_conditions( 126 self, match_criteria: list[tuple] | dict, exact_match: bool = True 127 ) -> str: 128 """Builds and returns the conditional portion of a query. 129 130 #### :params: 131 132 `match_criteria`: Can be a list of 2-tuples where each 133 tuple is `(columnName, rowValue)` or a dictionary where 134 keys are column names and values are row values. 135 136 `exact_match`: If `False`, the row value for a given column 137 will be matched as a substring. 138 139 Usage e.g.: 140 141 >>> self.cursor.execute(f'select * from {table} where {conditions};')""" 142 if type(match_criteria) == dict: 143 match_criteria = [(k, v) for k, v in match_criteria.items()] 144 if exact_match: 145 conditions = " and ".join( 146 f'"{column_row[0]}" = "{column_row[1]}"' 147 for column_row in match_criteria 148 ) 149 else: 150 conditions = " and ".join( 151 f'"{column_row[0]}" like "%{column_row[1]}%"' 152 for column_row in match_criteria 153 ) 154 return f"({conditions})" 155 156 @_connect 157 def query(self, query_) -> list[Any]: 158 """Execute an arbitrary query and return the results.""" 159 self.cursor.execute(query_) 160 return self.cursor.fetchall() 161 162 @_connect 163 def create_tables(self, table_defs: list[str] = []): 164 """Create tables if they don't exist. 165 166 :param `table_defs`: Each definition should be in the form `table_name(column_definitions)`""" 167 if len(table_defs) > 0: 168 table_names = self.get_table_names() 169 for table in table_defs: 170 if table.split("(")[0].strip() not in table_names: 171 self.cursor.execute(f"create table {table};") 172 self.logger.info(f'{table.split("(")[0]} table created.') 173 174 @_connect 175 def create_table(self, table: str, column_defs: list[str]): 176 """Create a table if it doesn't exist. 177 178 #### :params: 179 180 `table`: Name of the table to create. 181 182 `column_defs`: List of column definitions in proper Sqlite3 sytax. 183 i.e. `"column_name text unique"` or `"column_name int primary key"` etc.""" 184 if table not in self.get_table_names(): 185 query = f"create table {table}({', '.join(column_defs)});" 186 self.cursor.execute(query) 187 self.logger.info(f"'{table}' table created.") 188 189 @_connect 190 def get_table_names(self) -> list[str]: 191 """Returns a list of table names from the database.""" 192 self.cursor.execute( 193 'select name from sqlite_Schema where type = "table" and name not like "sqlite_%";' 194 ) 195 return [result[0] for result in self.cursor.fetchall()] 196 197 @_connect 198 def get_column_names(self, table: str) -> list[str]: 199 """Return a list of column names from a table.""" 200 self.cursor.execute(f"select * from {table} where 1=0") 201 return [description[0] for description in self.cursor.description] 202 203 @_connect 204 def count( 205 self, 206 table: str, 207 match_criteria: list[tuple] | dict | None = None, 208 exact_match: bool = True, 209 ) -> int: 210 """Return number of items in `table`. 211 212 #### :params: 213 214 `match_criteria`: Can be a list of 2-tuples where each 215 tuple is `(columnName, rowValue)` or a dictionary where 216 keys are column names and values are row values. 217 If `None`, all rows from the table will be counted. 218 219 `exact_match`: If `False`, the row value for a given column 220 in `match_criteria` will be matched as a substring. 221 Has no effect if `match_criteria` is `None`. 222 """ 223 query = f"select count(_rowid_) from {table}" 224 try: 225 if match_criteria: 226 self.cursor.execute( 227 f"{query} where {self._get_conditions(match_criteria, exact_match)};" 228 ) 229 else: 230 self.cursor.execute(f"{query}") 231 return self.cursor.fetchone()[0] 232 except: 233 return 0 234 235 @_connect 236 def add_row( 237 self, table: str, values: tuple[Any], columns: tuple[str] | None = None 238 ): 239 """Add a row of values to a table. 240 241 #### :params: 242 243 `table`: The table to insert values into. 244 245 `values`: A tuple of values to be inserted into the table. 246 247 `columns`: If `None`, `values` is expected to supply a value for every column in the table. 248 If `columns` is provided, it should contain the same number of elements as `values`.""" 249 parameterizer = ", ".join("?" for _ in values) 250 logger_values = ", ".join(str(value) for value in values) 251 try: 252 if columns: 253 columns_query = ", ".join(column for column in columns) 254 self.cursor.execute( 255 f"insert into {table} ({columns_query}) values({parameterizer});", 256 values, 257 ) 258 else: 259 self.cursor.execute( 260 f"insert into {table} values({parameterizer});", values 261 ) 262 self.logger.info(f'Added "{logger_values}" to {table} table.') 263 except Exception as e: 264 if "constraint" not in str(e).lower(): 265 self.logger.exception( 266 f'Error adding "{logger_values}" to {table} table.' 267 ) 268 else: 269 self.logger.debug(str(e)) 270 271 @_connect 272 def get_rows( 273 self, 274 table: str, 275 match_criteria: list[tuple] | dict | None = None, 276 exact_match: bool = True, 277 sort_by_column: str | None = None, 278 columns_to_return: list[str] | None = None, 279 return_as_dataframe: bool = False, 280 values_only: bool = False, 281 order_by: str | None = None, 282 limit: str | int | None = None, 283 ) -> list[dict] | list[tuple] | pandas.DataFrame: 284 """Return matching rows from `table`. 285 286 By default, rows will be returned as a list of dictionaries of the form `[{"column_name": value, ...}, ...]` 287 288 289 #### :params: 290 291 `match_criteria`: Can be a list of 2-tuples where each 292 tuple is `(columnName, rowValue)` or a dictionary where 293 keys are column names and values are row values. 294 295 `exact_match`: If `False`, the row value for a given column will be matched as a substring. 296 297 `sort_by_column`: A column name to sort the results by. 298 This will sort results in Python after retrieving them from the db. 299 Use the 'order_by' param to use SQLite engine for ordering. 300 301 `columns_to_return`: Optional list of column names. 302 If provided, the elements returned by this function will only contain the provided columns. 303 Otherwise every column in the row is returned. 304 305 `return_as_dataframe`: Return the results as a `pandas.DataFrame` object. 306 307 `values_only`: Return the results as a list of tuples. 308 309 `order_by`: If given, a `order by {order_by}` clause will be added to the select query. 310 311 `limit`: If given, a `limit {limit}` clause will be added to the select query. 312 """ 313 314 if type(columns_to_return) is str: 315 columns_to_return = [columns_to_return] 316 query = f"select * from {table}" 317 matches = [] 318 if match_criteria: 319 query += f" where {self._get_conditions(match_criteria, exact_match)}" 320 if order_by: 321 query += f" order by {order_by}" 322 if limit: 323 query += f" limit {limit}" 324 query += ";" 325 self.cursor.execute(query) 326 matches = self.cursor.fetchall() 327 results = [self._get_dict(table, match, columns_to_return) for match in matches] 328 if sort_by_column: 329 results = sorted(results, key=lambda x: x[sort_by_column]) 330 if return_as_dataframe: 331 return pandas.DataFrame(results) 332 if values_only: 333 return [tuple(row.values()) for row in results] 334 else: 335 return results 336 337 @_connect 338 def find( 339 self, table: str, query_string: str, columns: list[str] | None = None 340 ) -> list[dict]: 341 """Search for rows that contain `query_string` as a substring of any column. 342 343 #### :params: 344 345 `table`: The table to search. 346 347 `query_string`: The substring to search for in all columns. 348 349 `columns`: A list of columns to search for query_string. 350 If None, all columns in the table will be searched. 351 """ 352 if type(columns) is str: 353 columns = [columns] 354 results = [] 355 if not columns: 356 columns = self.get_column_names(table) 357 for column in columns: 358 results.extend( 359 [ 360 row 361 for row in self.get_rows( 362 table, [(column, query_string)], exact_match=False 363 ) 364 if row not in results 365 ] 366 ) 367 return results 368 369 @_connect 370 def delete( 371 self, table: str, match_criteria: list[tuple] | dict, exact_match: bool = True 372 ) -> int: 373 """Delete records from `table`. 374 375 Returns the number of deleted records. 376 377 #### :params: 378 379 `match_criteria`: Can be a list of 2-tuples where each tuple is `(column_name, value)` 380 or a dictionary where keys are column names and values are corresponding values. 381 382 `exact_match`: If `False`, the value for a given column will be matched as a substring. 383 """ 384 num_matches = self.count(table, match_criteria, exact_match) 385 conditions = self._get_conditions(match_criteria, exact_match) 386 try: 387 self.cursor.execute(f"delete from {table} where {conditions};") 388 self.logger.info( 389 f'Deleted {num_matches} from "{table}" where {conditions}".' 390 ) 391 return num_matches 392 except Exception as e: 393 self.logger.debug(f'Error deleting from "{table}" where {conditions}.\n{e}') 394 return 0 395 396 @_connect 397 def update( 398 self, 399 table: str, 400 column_to_update: str, 401 new_value: Any, 402 match_criteria: list[tuple] | dict | None = None, 403 ) -> bool: 404 """Update the value in `column_to_update` to `new_value` for rows matched with `match_criteria`. 405 406 #### :params: 407 408 `table`: The table to update rows in. 409 410 `column_to_update`: The column to be updated in the matched rows. 411 412 `new_value`: The new value to insert. 413 414 `match_criteria`: Can be a list of 2-tuples where each tuple is `(columnName, rowValue)` 415 or a dictionary where keys are column names and values are corresponding values. 416 If `None`, every row in `table` will be updated. 417 418 Returns `True` if successful, `False` if not.""" 419 query = f"update {table} set {column_to_update} = ?" 420 conditions = "" 421 if match_criteria: 422 if self.count(table, match_criteria) == 0: 423 self.logger.info( 424 f"Couldn't find matching records in {table} table to update to '{new_value}'" 425 ) 426 return False 427 conditions = self._get_conditions(match_criteria) 428 query += f" where {conditions}" 429 else: 430 conditions = None 431 query += ";" 432 try: 433 self.cursor.execute( 434 query, 435 (new_value,), 436 ) 437 self.logger.info( 438 f'Updated "{column_to_update}" in "{table}" table to "{new_value}" where {conditions}' 439 ) 440 return True 441 except Exception as e: 442 self.logger.error( 443 f'Failed to update "{column_to_update}" in "{table}" table to "{new_value}" where {conditions}"\n{e}' 444 ) 445 return False 446 447 @_connect 448 def drop_table(self, table: str) -> bool: 449 """Drop `table` from the database. 450 451 Returns `True` if successful, `False` if not.""" 452 try: 453 self.cursor.execute(f"drop Table {table};") 454 self.logger.info(f'Dropped table "{table}"') 455 return True 456 except Exception as e: 457 print(e) 458 self.logger.error(f'Failed to drop table "{table}"') 459 return False 460 461 @_connect 462 def add_column( 463 self, table: str, column: str, _type: str, default_value: str | None = None 464 ): 465 """Add a new column to `table`. 466 467 #### :params: 468 469 `column`: Name of the column to add. 470 471 `_type`: The data type of the new column. 472 473 `default_value`: Optional default value for the column.""" 474 try: 475 if default_value: 476 self.cursor.execute( 477 f"alter table {table} add column {column} {_type} default {default_value};" 478 ) 479 self.update(table, column, default_value) 480 else: 481 self.cursor.execute(f"alter table {table} add column {column} {_type};") 482 self.logger.info(f'Added column "{column}" to "{table}" table.') 483 except Exception as e: 484 self.logger.error(f'Failed to add column "{column}" to "{table}" table.') 485 486 @staticmethod 487 def data_to_string( 488 data: list[dict], sort_key: str | None = None, wrap_to_terminal: bool = True 489 ) -> str: 490 """Uses tabulate to produce pretty string output from a list of dictionaries. 491 492 #### :params: 493 494 `data`: The list of dictionaries to create a grid from. 495 Assumes all dictionaries in list have the same set of keys. 496 497 `sort_key`: Optional dictionary key to sort data with. 498 499 `wrap_to_terminal`: If `True`, the table width will be wrapped to fit within the current terminal window. 500 Pass as `False` if the output is going into something like a `.txt` file.""" 501 return data_to_string(data, sort_key, wrap_to_terminal)
Sqli wrapper so queries don't need to be written except table definitions.
Supports saving and reading dates as datetime objects.
Supports using a context manager.
34 def __init__( 35 self, 36 dbpath: str | Pathier, 37 logger_encoding: str = "utf-8", 38 logger_message_format: str = "{levelname}|-|{asctime}|-|{message}", 39 ): 40 """ 41 #### :params: 42 43 `dbpath`: String or Path object to database file. 44 If a relative path is given, it will be relative to the 45 current working directory. The log file will be saved to the 46 same directory. 47 48 `logger_message_format`: '{' style format string for the logger object.""" 49 self.dbpath = Pathier(dbpath) 50 self.dbname = Pathier(dbpath).name 51 self.dbpath.parent.mkdir(parents=True, exist_ok=True) 52 self._logger_init( 53 encoding=logger_encoding, message_format=logger_message_format 54 ) 55 self.connection_open = False
:params:
dbpath
: String or Path object to database file.
If a relative path is given, it will be relative to the
current working directory. The log file will be saved to the
same directory.
logger_message_format
: '{' style format string for the logger object.
64 def open(self): 65 """Open connection to db.""" 66 self.connection = sqlite3.connect( 67 self.dbpath, 68 detect_types=sqlite3.PARSE_DECLTYPES | sqlite3.PARSE_COLNAMES, 69 timeout=10, 70 ) 71 self.connection.execute("pragma foreign_keys = 1;") 72 self.cursor = self.connection.cursor() 73 self.connection_open = True
Open connection to db.
75 def close(self): 76 """Save and close connection to db. 77 78 Call this as soon as you are done using the database if you have 79 multiple threads or processes using the same database.""" 80 if self.connection_open: 81 self.connection.commit() 82 self.connection.close() 83 self.connection_open = False
Save and close connection to db.
Call this as soon as you are done using the database if you have multiple threads or processes using the same database.
156 @_connect 157 def query(self, query_) -> list[Any]: 158 """Execute an arbitrary query and return the results.""" 159 self.cursor.execute(query_) 160 return self.cursor.fetchall()
Execute an arbitrary query and return the results.
162 @_connect 163 def create_tables(self, table_defs: list[str] = []): 164 """Create tables if they don't exist. 165 166 :param `table_defs`: Each definition should be in the form `table_name(column_definitions)`""" 167 if len(table_defs) > 0: 168 table_names = self.get_table_names() 169 for table in table_defs: 170 if table.split("(")[0].strip() not in table_names: 171 self.cursor.execute(f"create table {table};") 172 self.logger.info(f'{table.split("(")[0]} table created.')
Create tables if they don't exist.
Parameters
table_defs
: Each definition should be in the formtable_name(column_definitions)
174 @_connect 175 def create_table(self, table: str, column_defs: list[str]): 176 """Create a table if it doesn't exist. 177 178 #### :params: 179 180 `table`: Name of the table to create. 181 182 `column_defs`: List of column definitions in proper Sqlite3 sytax. 183 i.e. `"column_name text unique"` or `"column_name int primary key"` etc.""" 184 if table not in self.get_table_names(): 185 query = f"create table {table}({', '.join(column_defs)});" 186 self.cursor.execute(query) 187 self.logger.info(f"'{table}' table created.")
Create a table if it doesn't exist.
:params:
table
: Name of the table to create.
column_defs
: List of column definitions in proper Sqlite3 sytax.
i.e. "column_name text unique"
or "column_name int primary key"
etc.
189 @_connect 190 def get_table_names(self) -> list[str]: 191 """Returns a list of table names from the database.""" 192 self.cursor.execute( 193 'select name from sqlite_Schema where type = "table" and name not like "sqlite_%";' 194 ) 195 return [result[0] for result in self.cursor.fetchall()]
Returns a list of table names from the database.
197 @_connect 198 def get_column_names(self, table: str) -> list[str]: 199 """Return a list of column names from a table.""" 200 self.cursor.execute(f"select * from {table} where 1=0") 201 return [description[0] for description in self.cursor.description]
Return a list of column names from a table.
203 @_connect 204 def count( 205 self, 206 table: str, 207 match_criteria: list[tuple] | dict | None = None, 208 exact_match: bool = True, 209 ) -> int: 210 """Return number of items in `table`. 211 212 #### :params: 213 214 `match_criteria`: Can be a list of 2-tuples where each 215 tuple is `(columnName, rowValue)` or a dictionary where 216 keys are column names and values are row values. 217 If `None`, all rows from the table will be counted. 218 219 `exact_match`: If `False`, the row value for a given column 220 in `match_criteria` will be matched as a substring. 221 Has no effect if `match_criteria` is `None`. 222 """ 223 query = f"select count(_rowid_) from {table}" 224 try: 225 if match_criteria: 226 self.cursor.execute( 227 f"{query} where {self._get_conditions(match_criteria, exact_match)};" 228 ) 229 else: 230 self.cursor.execute(f"{query}") 231 return self.cursor.fetchone()[0] 232 except: 233 return 0
Return number of items in table
.
:params:
match_criteria
: Can be a list of 2-tuples where each
tuple is (columnName, rowValue)
or a dictionary where
keys are column names and values are row values.
If None
, all rows from the table will be counted.
exact_match
: If False
, the row value for a given column
in match_criteria
will be matched as a substring.
Has no effect if match_criteria
is None
.
235 @_connect 236 def add_row( 237 self, table: str, values: tuple[Any], columns: tuple[str] | None = None 238 ): 239 """Add a row of values to a table. 240 241 #### :params: 242 243 `table`: The table to insert values into. 244 245 `values`: A tuple of values to be inserted into the table. 246 247 `columns`: If `None`, `values` is expected to supply a value for every column in the table. 248 If `columns` is provided, it should contain the same number of elements as `values`.""" 249 parameterizer = ", ".join("?" for _ in values) 250 logger_values = ", ".join(str(value) for value in values) 251 try: 252 if columns: 253 columns_query = ", ".join(column for column in columns) 254 self.cursor.execute( 255 f"insert into {table} ({columns_query}) values({parameterizer});", 256 values, 257 ) 258 else: 259 self.cursor.execute( 260 f"insert into {table} values({parameterizer});", values 261 ) 262 self.logger.info(f'Added "{logger_values}" to {table} table.') 263 except Exception as e: 264 if "constraint" not in str(e).lower(): 265 self.logger.exception( 266 f'Error adding "{logger_values}" to {table} table.' 267 ) 268 else: 269 self.logger.debug(str(e))
Add a row of values to a table.
:params:
table
: The table to insert values into.
values
: A tuple of values to be inserted into the table.
columns
: If None
, values
is expected to supply a value for every column in the table.
If columns
is provided, it should contain the same number of elements as values
.
271 @_connect 272 def get_rows( 273 self, 274 table: str, 275 match_criteria: list[tuple] | dict | None = None, 276 exact_match: bool = True, 277 sort_by_column: str | None = None, 278 columns_to_return: list[str] | None = None, 279 return_as_dataframe: bool = False, 280 values_only: bool = False, 281 order_by: str | None = None, 282 limit: str | int | None = None, 283 ) -> list[dict] | list[tuple] | pandas.DataFrame: 284 """Return matching rows from `table`. 285 286 By default, rows will be returned as a list of dictionaries of the form `[{"column_name": value, ...}, ...]` 287 288 289 #### :params: 290 291 `match_criteria`: Can be a list of 2-tuples where each 292 tuple is `(columnName, rowValue)` or a dictionary where 293 keys are column names and values are row values. 294 295 `exact_match`: If `False`, the row value for a given column will be matched as a substring. 296 297 `sort_by_column`: A column name to sort the results by. 298 This will sort results in Python after retrieving them from the db. 299 Use the 'order_by' param to use SQLite engine for ordering. 300 301 `columns_to_return`: Optional list of column names. 302 If provided, the elements returned by this function will only contain the provided columns. 303 Otherwise every column in the row is returned. 304 305 `return_as_dataframe`: Return the results as a `pandas.DataFrame` object. 306 307 `values_only`: Return the results as a list of tuples. 308 309 `order_by`: If given, a `order by {order_by}` clause will be added to the select query. 310 311 `limit`: If given, a `limit {limit}` clause will be added to the select query. 312 """ 313 314 if type(columns_to_return) is str: 315 columns_to_return = [columns_to_return] 316 query = f"select * from {table}" 317 matches = [] 318 if match_criteria: 319 query += f" where {self._get_conditions(match_criteria, exact_match)}" 320 if order_by: 321 query += f" order by {order_by}" 322 if limit: 323 query += f" limit {limit}" 324 query += ";" 325 self.cursor.execute(query) 326 matches = self.cursor.fetchall() 327 results = [self._get_dict(table, match, columns_to_return) for match in matches] 328 if sort_by_column: 329 results = sorted(results, key=lambda x: x[sort_by_column]) 330 if return_as_dataframe: 331 return pandas.DataFrame(results) 332 if values_only: 333 return [tuple(row.values()) for row in results] 334 else: 335 return results
Return matching rows from table
.
By default, rows will be returned as a list of dictionaries of the form [{"column_name": value, ...}, ...]
:params:
match_criteria
: Can be a list of 2-tuples where each
tuple is (columnName, rowValue)
or a dictionary where
keys are column names and values are row values.
exact_match
: If False
, the row value for a given column will be matched as a substring.
sort_by_column
: A column name to sort the results by.
This will sort results in Python after retrieving them from the db.
Use the 'order_by' param to use SQLite engine for ordering.
columns_to_return
: Optional list of column names.
If provided, the elements returned by this function will only contain the provided columns.
Otherwise every column in the row is returned.
return_as_dataframe
: Return the results as a pandas.DataFrame
object.
values_only
: Return the results as a list of tuples.
order_by
: If given, a order by {order_by}
clause will be added to the select query.
limit
: If given, a limit {limit}
clause will be added to the select query.
337 @_connect 338 def find( 339 self, table: str, query_string: str, columns: list[str] | None = None 340 ) -> list[dict]: 341 """Search for rows that contain `query_string` as a substring of any column. 342 343 #### :params: 344 345 `table`: The table to search. 346 347 `query_string`: The substring to search for in all columns. 348 349 `columns`: A list of columns to search for query_string. 350 If None, all columns in the table will be searched. 351 """ 352 if type(columns) is str: 353 columns = [columns] 354 results = [] 355 if not columns: 356 columns = self.get_column_names(table) 357 for column in columns: 358 results.extend( 359 [ 360 row 361 for row in self.get_rows( 362 table, [(column, query_string)], exact_match=False 363 ) 364 if row not in results 365 ] 366 ) 367 return results
Search for rows that contain query_string
as a substring of any column.
:params:
table
: The table to search.
query_string
: The substring to search for in all columns.
columns
: A list of columns to search for query_string.
If None, all columns in the table will be searched.
369 @_connect 370 def delete( 371 self, table: str, match_criteria: list[tuple] | dict, exact_match: bool = True 372 ) -> int: 373 """Delete records from `table`. 374 375 Returns the number of deleted records. 376 377 #### :params: 378 379 `match_criteria`: Can be a list of 2-tuples where each tuple is `(column_name, value)` 380 or a dictionary where keys are column names and values are corresponding values. 381 382 `exact_match`: If `False`, the value for a given column will be matched as a substring. 383 """ 384 num_matches = self.count(table, match_criteria, exact_match) 385 conditions = self._get_conditions(match_criteria, exact_match) 386 try: 387 self.cursor.execute(f"delete from {table} where {conditions};") 388 self.logger.info( 389 f'Deleted {num_matches} from "{table}" where {conditions}".' 390 ) 391 return num_matches 392 except Exception as e: 393 self.logger.debug(f'Error deleting from "{table}" where {conditions}.\n{e}') 394 return 0
Delete records from table
.
Returns the number of deleted records.
:params:
match_criteria
: Can be a list of 2-tuples where each tuple is (column_name, value)
or a dictionary where keys are column names and values are corresponding values.
exact_match
: If False
, the value for a given column will be matched as a substring.
396 @_connect 397 def update( 398 self, 399 table: str, 400 column_to_update: str, 401 new_value: Any, 402 match_criteria: list[tuple] | dict | None = None, 403 ) -> bool: 404 """Update the value in `column_to_update` to `new_value` for rows matched with `match_criteria`. 405 406 #### :params: 407 408 `table`: The table to update rows in. 409 410 `column_to_update`: The column to be updated in the matched rows. 411 412 `new_value`: The new value to insert. 413 414 `match_criteria`: Can be a list of 2-tuples where each tuple is `(columnName, rowValue)` 415 or a dictionary where keys are column names and values are corresponding values. 416 If `None`, every row in `table` will be updated. 417 418 Returns `True` if successful, `False` if not.""" 419 query = f"update {table} set {column_to_update} = ?" 420 conditions = "" 421 if match_criteria: 422 if self.count(table, match_criteria) == 0: 423 self.logger.info( 424 f"Couldn't find matching records in {table} table to update to '{new_value}'" 425 ) 426 return False 427 conditions = self._get_conditions(match_criteria) 428 query += f" where {conditions}" 429 else: 430 conditions = None 431 query += ";" 432 try: 433 self.cursor.execute( 434 query, 435 (new_value,), 436 ) 437 self.logger.info( 438 f'Updated "{column_to_update}" in "{table}" table to "{new_value}" where {conditions}' 439 ) 440 return True 441 except Exception as e: 442 self.logger.error( 443 f'Failed to update "{column_to_update}" in "{table}" table to "{new_value}" where {conditions}"\n{e}' 444 ) 445 return False
Update the value in column_to_update
to new_value
for rows matched with match_criteria
.
:params:
table
: The table to update rows in.
column_to_update
: The column to be updated in the matched rows.
new_value
: The new value to insert.
match_criteria
: Can be a list of 2-tuples where each tuple is (columnName, rowValue)
or a dictionary where keys are column names and values are corresponding values.
If None
, every row in table
will be updated.
Returns True
if successful, False
if not.
447 @_connect 448 def drop_table(self, table: str) -> bool: 449 """Drop `table` from the database. 450 451 Returns `True` if successful, `False` if not.""" 452 try: 453 self.cursor.execute(f"drop Table {table};") 454 self.logger.info(f'Dropped table "{table}"') 455 return True 456 except Exception as e: 457 print(e) 458 self.logger.error(f'Failed to drop table "{table}"') 459 return False
Drop table
from the database.
Returns True
if successful, False
if not.
461 @_connect 462 def add_column( 463 self, table: str, column: str, _type: str, default_value: str | None = None 464 ): 465 """Add a new column to `table`. 466 467 #### :params: 468 469 `column`: Name of the column to add. 470 471 `_type`: The data type of the new column. 472 473 `default_value`: Optional default value for the column.""" 474 try: 475 if default_value: 476 self.cursor.execute( 477 f"alter table {table} add column {column} {_type} default {default_value};" 478 ) 479 self.update(table, column, default_value) 480 else: 481 self.cursor.execute(f"alter table {table} add column {column} {_type};") 482 self.logger.info(f'Added column "{column}" to "{table}" table.') 483 except Exception as e: 484 self.logger.error(f'Failed to add column "{column}" to "{table}" table.')
Add a new column to table
.
:params:
column
: Name of the column to add.
_type
: The data type of the new column.
default_value
: Optional default value for the column.
486 @staticmethod 487 def data_to_string( 488 data: list[dict], sort_key: str | None = None, wrap_to_terminal: bool = True 489 ) -> str: 490 """Uses tabulate to produce pretty string output from a list of dictionaries. 491 492 #### :params: 493 494 `data`: The list of dictionaries to create a grid from. 495 Assumes all dictionaries in list have the same set of keys. 496 497 `sort_key`: Optional dictionary key to sort data with. 498 499 `wrap_to_terminal`: If `True`, the table width will be wrapped to fit within the current terminal window. 500 Pass as `False` if the output is going into something like a `.txt` file.""" 501 return data_to_string(data, sort_key, wrap_to_terminal)
Uses tabulate to produce pretty string output from a list of dictionaries.
:params:
data
: The list of dictionaries to create a grid from.
Assumes all dictionaries in list have the same set of keys.
sort_key
: Optional dictionary key to sort data with.
wrap_to_terminal
: If True
, the table width will be wrapped to fit within the current terminal window.
Pass as False
if the output is going into something like a .txt
file.
504def data_to_string( 505 data: list[dict], sort_key: str | None = None, wrap_to_terminal: bool = True 506) -> str: 507 """Uses tabulate to produce pretty string output from a list of dictionaries. 508 509 #### :params: 510 511 `data`: The list of dictionaries to create a grid from. 512 Assumes all dictionaries in list have the same set of keys. 513 514 `sort_key`: Optional dictionary key to sort data with. 515 516 `wrap_to_terminal`: If `True`, the table width will be wrapped to fit within the current terminal window. 517 Pass as `False` if the output is going into something like a `.txt` file.""" 518 if len(data) == 0: 519 return "" 520 if sort_key: 521 data = sorted(data, key=lambda d: d[sort_key]) 522 for i, d in enumerate(data): 523 for k in d: 524 data[i][k] = str(data[i][k]) 525 526 too_wide = True 527 terminal_width = os.get_terminal_size().columns 528 max_col_widths = terminal_width 529 # Make an output with effectively unrestricted column widths 530 # to see if shrinking is necessary 531 output = tabulate( 532 data, 533 headers="keys", 534 disable_numparse=True, 535 tablefmt="grid", 536 maxcolwidths=max_col_widths, 537 ) 538 current_width = output.index("\n") 539 if current_width < terminal_width: 540 too_wide = False 541 if wrap_to_terminal and too_wide: 542 print("Resizing grid to fit within the terminal...\n") 543 previous_col_widths = max_col_widths 544 acceptable_width = terminal_width - 10 545 while too_wide and max_col_widths > 1: 546 if current_width >= terminal_width: 547 previous_col_widths = max_col_widths 548 max_col_widths = int(max_col_widths * 0.5) 549 elif current_width < terminal_width: 550 # Without lowering acceptable_width, this condition will cause infinite loop 551 if max_col_widths == previous_col_widths - 1: 552 acceptable_width -= 10 553 max_col_widths = int( 554 max_col_widths + (0.5 * (previous_col_widths - max_col_widths)) 555 ) 556 output = tabulate( 557 data, 558 headers="keys", 559 disable_numparse=True, 560 tablefmt="grid", 561 maxcolwidths=max_col_widths, 562 ) 563 current_width = output.index("\n") 564 if acceptable_width < current_width < terminal_width: 565 too_wide = False 566 if too_wide: 567 print("Couldn't resize grid to fit within the terminal.") 568 return str(data) 569 return output
Uses tabulate to produce pretty string output from a list of dictionaries.
:params:
data
: The list of dictionaries to create a grid from.
Assumes all dictionaries in list have the same set of keys.
sort_key
: Optional dictionary key to sort data with.
wrap_to_terminal
: If True
, the table width will be wrapped to fit within the current terminal window.
Pass as False
if the output is going into something like a .txt
file.