databased.databased
1import sqlite3 2from typing import Any 3 4import loggi 5from griddle import griddy 6from pathier import Pathier, Pathish 7 8 9def dict_factory(cursor: sqlite3.Cursor, row: tuple) -> dict: 10 fields = [column[0] for column in cursor.description] 11 return {column: value for column, value in zip(fields, row)} 12 13 14class Databased: 15 """SQLite3 wrapper.""" 16 17 def __init__( 18 self, 19 dbpath: Pathish = "db.sqlite3", 20 connection_timeout: float = 10, 21 detect_types: bool = True, 22 enforce_foreign_keys: bool = True, 23 commit_on_close: bool = True, 24 logger_encoding: str = "utf-8", 25 logger_message_format: str = "{levelname}|-|{asctime}|-|{message}", 26 ): 27 """ """ 28 self.path = dbpath 29 self.connection_timeout = connection_timeout 30 self.connection = None 31 self._logger_init(logger_message_format, logger_encoding) 32 self.detect_types = detect_types 33 self.commit_on_close = commit_on_close 34 self.enforce_foreign_keys = enforce_foreign_keys 35 36 def __enter__(self): 37 self.connect() 38 return self 39 40 def __exit__(self, *args, **kwargs): 41 self.close() 42 43 @property 44 def commit_on_close(self) -> bool: 45 """Should commit database before closing connection when `self.close()` is called.""" 46 return self._commit_on_close 47 48 @commit_on_close.setter 49 def commit_on_close(self, should_commit_on_close: bool): 50 self._commit_on_close = should_commit_on_close 51 52 @property 53 def connected(self) -> bool: 54 """Whether this `Databased` instance is connected to the database file or not.""" 55 return self.connection is not None 56 57 @property 58 def connection_timeout(self) -> float: 59 """Changes to this property won't take effect until the current connection, if open, is closed and a new connection opened.""" 60 return self._connection_timeout 61 62 @connection_timeout.setter 63 def connection_timeout(self, timeout: float): 64 self._connection_timeout = timeout 65 66 @property 67 def detect_types(self) -> bool: 68 """Should use `detect_types=sqlite3.PARSE_DECLTYPES | sqlite3.PARSE_COLNAMES` when establishing a database connection. 69 70 Changes to this property won't take effect until the current connection, if open, is closed and a new connection opened. 71 """ 72 return self._detect_types 73 74 @detect_types.setter 75 def detect_types(self, should_detect: bool): 76 self._detect_types = should_detect 77 78 @property 79 def enforce_foreign_keys(self) -> bool: 80 return self._enforce_foreign_keys 81 82 @enforce_foreign_keys.setter 83 def enforce_foreign_keys(self, should_enforce: bool): 84 self._enforce_foreign_keys = should_enforce 85 self._set_foreign_key_enforcement() 86 87 @property 88 def indicies(self) -> list[str]: 89 """List of indicies for this database.""" 90 return [ 91 table["name"] 92 for table in self.query( 93 "SELECT name FROM sqlite_Schema WHERE type = 'index';" 94 ) 95 ] 96 97 @property 98 def name(self) -> str: 99 """The name of this database.""" 100 return self.path.stem 101 102 @property 103 def path(self) -> Pathier: 104 """The path to this database file.""" 105 return self._path 106 107 @path.setter 108 def path(self, new_path: Pathish): 109 """If `new_path` doesn't exist, it will be created (including parent folders).""" 110 self._path = Pathier(new_path) 111 if not self.path.exists(): 112 self.path.touch() 113 114 @property 115 def tables(self) -> list[str]: 116 """List of table names for this database.""" 117 return [ 118 table["name"] 119 for table in self.query( 120 "SELECT name FROM sqlite_Schema WHERE type = 'table' AND name NOT LIKE 'sqlite_%';" 121 ) 122 ] 123 124 @property 125 def views(self) -> list[str]: 126 """List of view for this database.""" 127 return [ 128 table["name"] 129 for table in self.query( 130 "SELECT name FROM sqlite_Schema WHERE type = 'view' AND name NOT LIKE 'sqlite_%';" 131 ) 132 ] 133 134 def _logger_init(self, message_format: str, encoding: str): 135 """:param: `message_format`: `{` style format string.""" 136 self.logger = loggi.getLogger(self.name) 137 138 def _prepare_insert_queries( 139 self, table: str, columns: tuple[str, ...], values: list[tuple[Any, ...]] 140 ) -> list[tuple[str, tuple[Any, ...]]]: 141 """Format a list of insert statements. 142 143 The returned value is a list because `values` will be broken up into chunks. 144 145 Each list element is a two tuple consisting of the parameterized query string and a tuple of values.""" 146 inserts = [] 147 max_row_count = 900 148 column_list = "(" + ", ".join(columns) + ")" 149 for i in range(0, len(values), max_row_count): 150 chunk = values[i : i + max_row_count] 151 placeholder = ( 152 "(" + "),(".join((", ".join(("?" for _ in row)) for row in chunk)) + ")" 153 ) 154 flattened_values = tuple((value for row in chunk for value in row)) 155 inserts.append( 156 ( 157 f"INSERT INTO {table} {column_list} VALUES {placeholder};", 158 flattened_values, 159 ) 160 ) 161 return inserts 162 163 def _set_foreign_key_enforcement(self): 164 if self.connection: 165 self.connection.execute( 166 f"pragma foreign_keys = {int(self.enforce_foreign_keys)};" 167 ) 168 169 def add_column(self, table: str, column_def: str): 170 """Add a column to `table`. 171 172 `column_def` should be in the form `{column_name} {type_name} {constraint}`. 173 174 i.e. 175 >>> db = Databased() 176 >>> db.add_column("rides", "num_stops INTEGER NOT NULL DEFAULT 0")""" 177 self.query(f"ALTER TABLE {table} ADD {column_def};") 178 179 def close(self): 180 """Disconnect from the database. 181 182 Does not call `commit()` for you unless the `commit_on_close` property is set to `True`. 183 """ 184 if self.connection: 185 if self.commit_on_close: 186 self.commit() 187 self.connection.close() 188 self.connection = None 189 190 def commit(self): 191 """Commit state of database.""" 192 if self.connection: 193 self.connection.commit() 194 self.logger.info("Committed successfully.") 195 else: 196 raise RuntimeError( 197 "Databased.commit(): Can't commit db with no open connection." 198 ) 199 200 def connect(self): 201 """Connect to the database.""" 202 self.connection = sqlite3.connect( 203 self.path, 204 detect_types=sqlite3.PARSE_DECLTYPES | sqlite3.PARSE_COLNAMES 205 if self.detect_types 206 else 0, 207 timeout=self.connection_timeout, 208 ) 209 self._set_foreign_key_enforcement() 210 self.connection.row_factory = dict_factory 211 212 def count( 213 self, 214 table: str, 215 column: str = "*", 216 where: str | None = None, 217 distinct: bool = False, 218 ) -> int: 219 """Return number of matching rows in `table` table. 220 221 Equivalent to: 222 >>> SELECT COUNT({distinct} {column}) FROM {table} {where};""" 223 query = ( 224 f"SELECT COUNT( {('DISTINCT' if distinct else '')} {column}) FROM {table}" 225 ) 226 if where: 227 query += f" WHERE {where}" 228 query += ";" 229 return int(list(self.query(query)[0].values())[0]) 230 231 def create_table(self, table: str, *column_defs: str): 232 """Create a table if it doesn't exist. 233 234 #### :params: 235 236 `table`: Name of the table to create. 237 238 `column_defs`: Any number of column names and their definitions in proper Sqlite3 sytax. 239 i.e. `"column_name TEXT UNIQUE"` or `"column_name INTEGER PRIMARY KEY"` etc.""" 240 columns = ", ".join(column_defs) 241 result = self.query(f"CREATE TABLE IF NOT EXISTS {table} ({columns});") 242 self.logger.info(f"'{table}' table created.") 243 244 def delete(self, table: str, where: str | None = None) -> int: 245 """Delete rows from `table` that satisfy the given `where` clause. 246 247 If `where` is `None`, all rows will be deleted. 248 249 Returns the number of deleted rows. 250 251 e.g. 252 >>> db = Databased() 253 >>> db.delete("rides", "distance < 5 AND average_speed < 7")""" 254 try: 255 if where: 256 self.query(f"DELETE FROM {table} WHERE {where};") 257 else: 258 self.query(f"DELETE FROM {table};") 259 row_count = self.cursor.rowcount 260 self.logger.info( 261 f"Deleted {row_count} rows from '{table}' where '{where}'." 262 ) 263 return row_count 264 except Exception as e: 265 self.logger.exception( 266 f"Error deleting rows from '{table}' where '{where}'." 267 ) 268 raise e 269 270 def describe(self, table: str) -> list[dict]: 271 """Returns information about `table`.""" 272 return self.query(f"pragma table_info('{table}');") 273 274 def drop_column(self, table: str, column: str): 275 """Drop `column` from `table`.""" 276 self.query(f"ALTER TABLE {table} DROP {column};") 277 278 def drop_table(self, table: str) -> bool: 279 """Drop `table` from the database. 280 281 Returns `True` if successful, `False` if not.""" 282 try: 283 self.query(f"DROP TABLE {table};") 284 self.logger.info(f"Dropped table '{table}'.") 285 return True 286 except Exception as e: 287 print(f"{type(e).__name__}: {e}") 288 self.logger.error(f"Failed to drop table '{table}'.") 289 return False 290 291 def execute_script(self, path: Pathish, encoding: str = "utf-8") -> list[dict]: 292 """Execute sql script located at `path`.""" 293 if not self.connected: 294 self.connect() 295 assert self.connection 296 return self.connection.executescript( 297 Pathier(path).read_text(encoding) 298 ).fetchall() 299 300 def get_columns(self, table: str) -> tuple[str, ...]: 301 """Returns a list of column names in `table`.""" 302 return tuple( 303 (column["name"] for column in self.query(f"pragma table_info('{table}');")) 304 ) 305 306 def insert( 307 self, table: str, columns: tuple[str, ...], values: list[tuple[Any, ...]] 308 ) -> int: 309 """Insert rows of `values` into `columns` of `table`. 310 311 Each `tuple` in `values` corresponds to an individual row that is to be inserted. 312 """ 313 row_count = 0 314 for insert in self._prepare_insert_queries(table, columns, values): 315 try: 316 self.query(insert[0], insert[1]) 317 row_count += self.cursor.rowcount 318 self.logger.info(f"Inserted {row_count} rows into '{table}' table.") 319 except Exception as e: 320 self.logger.exception(f"Error inserting rows into '{table}' table.") 321 raise e 322 return row_count 323 324 def query(self, query_: str, parameters: tuple[Any, ...] = tuple()) -> list[dict]: 325 """Execute an SQL query and return the results. 326 327 Ensures that the database connection is opened before executing the command. 328 329 The cursor used to execute the query will be available through `self.cursor` until the next time `self.query()` is called. 330 """ 331 if not self.connected: 332 self.connect() 333 assert self.connection 334 self.cursor = self.connection.cursor() 335 self.cursor.execute(query_, parameters) 336 return self.cursor.fetchall() 337 338 def rename_column(self, table: str, column_to_rename: str, new_column_name: str): 339 """Rename a column in `table`.""" 340 self.query( 341 f"ALTER TABLE {table} RENAME {column_to_rename} TO {new_column_name};" 342 ) 343 344 def rename_table(self, table_to_rename: str, new_table_name: str): 345 """Rename a table.""" 346 self.query(f"ALTER TABLE {table_to_rename} RENAME TO {new_table_name};") 347 348 def select( 349 self, 350 table: str, 351 columns: list[str] = ["*"], 352 joins: list[str] | None = None, 353 where: str | None = None, 354 group_by: str | None = None, 355 having: str | None = None, 356 order_by: str | None = None, 357 limit: int | str | None = None, 358 ) -> list[dict]: 359 """Return rows for given criteria. 360 361 For complex queries, use the `databased.query()` method. 362 363 Parameters `where`, `group_by`, `having`, `order_by`, and `limit` should not have 364 their corresponding key word in their string, but should otherwise be valid SQL. 365 366 `joins` should contain their key word (`INNER JOIN`, `LEFT JOIN`) in addition to the rest of the sub-statement. 367 368 >>> Databased().select( 369 "bike_rides", 370 "id, date, distance, moving_time, AVG(distance/moving_time) as average_speed", 371 where="distance > 20", 372 order_by="distance", 373 desc=True, 374 limit=10 375 ) 376 executes the query: 377 >>> SELECT 378 id, date, distance, moving_time, AVG(distance/moving_time) as average_speed 379 FROM 380 bike_rides 381 WHERE 382 distance > 20 383 ORDER BY 384 distance DESC 385 Limit 10;""" 386 query = f"SELECT {', '.join(columns)} FROM {table}" 387 if joins: 388 query += f" {' '.join(joins)}" 389 if where: 390 query += f" WHERE {where}" 391 if group_by: 392 query += f" GROUP BY {group_by}" 393 if having: 394 query += f" HAVING {having}" 395 if order_by: 396 query += f" ORDER BY {order_by}" 397 if limit: 398 query += f" LIMIT {limit}" 399 query += ";" 400 rows = self.query(query) 401 return rows 402 403 @staticmethod 404 def to_grid(data: list[dict], shrink_to_terminal: bool = True) -> str: 405 """Returns a tabular grid from `data`. 406 407 If `shrink_to_terminal` is `True`, the column widths of the grid will be reduced to fit within the current terminal. 408 """ 409 return griddy(data, "keys", shrink_to_terminal) 410 411 def update( 412 self, table: str, column: str, value: Any, where: str | None = None 413 ) -> int: 414 """Update `column` of `table` to `value` for rows satisfying the conditions in `where`. 415 416 If `where` is `None` all rows will be updated. 417 418 Returns the number of updated rows. 419 420 e.g. 421 >>> db = Databased() 422 >>> db.update("rides", "elevation", 100, "elevation < 100")""" 423 try: 424 if where: 425 self.query(f"UPDATE {table} SET {column} = ? WHERE {where};", (value,)) 426 else: 427 self.query(f"UPDATE {table} SET {column} = ?;", (value,)) 428 row_count = self.cursor.rowcount 429 self.logger.info( 430 f"Updated {row_count} rows in '{table}' table to '{column}' = '{value}' where '{where}'." 431 ) 432 return row_count 433 except Exception as e: 434 self.logger.exception( 435 f"Failed to update rows in '{table}' table to '{column}' = '{value}' where '{where}'." 436 ) 437 raise e 438 439 def vacuum(self) -> int: 440 """Reduce disk size of database after row/table deletion. 441 442 Returns space freed up in bytes.""" 443 size = self.path.size 444 self.query("VACUUM;") 445 return size - self.path.size 446 447 # Seat ========================== Database Dump ========================================= 448 449 def _format_column_def(self, description: dict) -> str: 450 name = description["name"] 451 type_ = description["type"] 452 primary_key = bool(description["pk"]) 453 not_null = bool(description["notnull"]) 454 default = description["dflt_value"] 455 column = f"{name} {type_}" 456 if primary_key: 457 column += f" PRIMARY KEY" 458 if not_null: 459 column += f" NOT NULL" 460 if default: 461 if isinstance(default, str): 462 default = f"{default}" 463 column += f" DEFAULT {default}" 464 return column 465 466 def _format_table_data(self, table: str) -> str: 467 columns = self.get_columns(table) 468 rows = [tuple(row.values()) for row in self.select(table)] 469 inserts = self._prepare_insert_queries(table, columns, rows) 470 insert_strings = [] 471 indent = " " * 4 472 for insert in inserts: 473 text = insert[0] 474 sub = "^$data$based$^" 475 text = text.replace("?", sub) 476 for value in insert[1]: 477 if not value: 478 value = "" 479 if isinstance(value, bool): 480 value = int(value) 481 if not isinstance(value, int) and (not isinstance(value, float)): 482 if isinstance(value, str): 483 value = value.replace('"', "'") 484 value = f'"{value}"' 485 text = text.replace(sub, str(value), 1) 486 for pair in [ 487 ("INSERT INTO ", f"INSERT INTO\n{indent}"), 488 (") VALUES (", f")\nVALUES\n{indent}("), 489 ("),", f"),\n{indent}"), 490 ]: 491 text = text.replace(pair[0], pair[1]) 492 insert_strings.append(text) 493 return "\n".join(insert_strings) 494 495 def _format_table_def(self, table: str) -> str: 496 description = self.describe(table) 497 indent = " " * 4 498 columns = ",\n".join( 499 (f"{indent * 2}{self._format_column_def(column)}" for column in description) 500 ) 501 table_def = ( 502 "CREATE TABLE IF NOT EXISTS\n" 503 + f"{indent}{table} (\n" 504 + columns 505 + f"\n{indent});" 506 ) 507 return table_def 508 509 def _get_data_dump_string(self, tables: list[str]) -> str: 510 return "\n\n".join((self._format_table_data(table) for table in tables)) 511 512 def _get_schema_dump_string(self, tables: list[str]) -> str: 513 return "\n\n".join((self._format_table_def(table) for table in tables)) 514 515 def dump_data(self, path: Pathish, tables: list[str] | None = None): 516 """Create a data dump file for the specified tables or all tables, if none are given.""" 517 tables = tables or self.tables 518 path = Pathier(path) 519 path.write_text(self._get_data_dump_string(tables), encoding="utf-8") 520 521 def dump_schema(self, path: Pathish, tables: list[str] | None = None): 522 """Create a schema dump file for the specified tables or all tables, if none are given. 523 524 NOTE: Foreign key relationships/constraints are not preserved when dumping the schema.""" 525 tables = tables or self.tables 526 path = Pathier(path) 527 path.write_text(self._get_schema_dump_string(tables), encoding="utf-8")
15class Databased: 16 """SQLite3 wrapper.""" 17 18 def __init__( 19 self, 20 dbpath: Pathish = "db.sqlite3", 21 connection_timeout: float = 10, 22 detect_types: bool = True, 23 enforce_foreign_keys: bool = True, 24 commit_on_close: bool = True, 25 logger_encoding: str = "utf-8", 26 logger_message_format: str = "{levelname}|-|{asctime}|-|{message}", 27 ): 28 """ """ 29 self.path = dbpath 30 self.connection_timeout = connection_timeout 31 self.connection = None 32 self._logger_init(logger_message_format, logger_encoding) 33 self.detect_types = detect_types 34 self.commit_on_close = commit_on_close 35 self.enforce_foreign_keys = enforce_foreign_keys 36 37 def __enter__(self): 38 self.connect() 39 return self 40 41 def __exit__(self, *args, **kwargs): 42 self.close() 43 44 @property 45 def commit_on_close(self) -> bool: 46 """Should commit database before closing connection when `self.close()` is called.""" 47 return self._commit_on_close 48 49 @commit_on_close.setter 50 def commit_on_close(self, should_commit_on_close: bool): 51 self._commit_on_close = should_commit_on_close 52 53 @property 54 def connected(self) -> bool: 55 """Whether this `Databased` instance is connected to the database file or not.""" 56 return self.connection is not None 57 58 @property 59 def connection_timeout(self) -> float: 60 """Changes to this property won't take effect until the current connection, if open, is closed and a new connection opened.""" 61 return self._connection_timeout 62 63 @connection_timeout.setter 64 def connection_timeout(self, timeout: float): 65 self._connection_timeout = timeout 66 67 @property 68 def detect_types(self) -> bool: 69 """Should use `detect_types=sqlite3.PARSE_DECLTYPES | sqlite3.PARSE_COLNAMES` when establishing a database connection. 70 71 Changes to this property won't take effect until the current connection, if open, is closed and a new connection opened. 72 """ 73 return self._detect_types 74 75 @detect_types.setter 76 def detect_types(self, should_detect: bool): 77 self._detect_types = should_detect 78 79 @property 80 def enforce_foreign_keys(self) -> bool: 81 return self._enforce_foreign_keys 82 83 @enforce_foreign_keys.setter 84 def enforce_foreign_keys(self, should_enforce: bool): 85 self._enforce_foreign_keys = should_enforce 86 self._set_foreign_key_enforcement() 87 88 @property 89 def indicies(self) -> list[str]: 90 """List of indicies for this database.""" 91 return [ 92 table["name"] 93 for table in self.query( 94 "SELECT name FROM sqlite_Schema WHERE type = 'index';" 95 ) 96 ] 97 98 @property 99 def name(self) -> str: 100 """The name of this database.""" 101 return self.path.stem 102 103 @property 104 def path(self) -> Pathier: 105 """The path to this database file.""" 106 return self._path 107 108 @path.setter 109 def path(self, new_path: Pathish): 110 """If `new_path` doesn't exist, it will be created (including parent folders).""" 111 self._path = Pathier(new_path) 112 if not self.path.exists(): 113 self.path.touch() 114 115 @property 116 def tables(self) -> list[str]: 117 """List of table names for this database.""" 118 return [ 119 table["name"] 120 for table in self.query( 121 "SELECT name FROM sqlite_Schema WHERE type = 'table' AND name NOT LIKE 'sqlite_%';" 122 ) 123 ] 124 125 @property 126 def views(self) -> list[str]: 127 """List of view for this database.""" 128 return [ 129 table["name"] 130 for table in self.query( 131 "SELECT name FROM sqlite_Schema WHERE type = 'view' AND name NOT LIKE 'sqlite_%';" 132 ) 133 ] 134 135 def _logger_init(self, message_format: str, encoding: str): 136 """:param: `message_format`: `{` style format string.""" 137 self.logger = loggi.getLogger(self.name) 138 139 def _prepare_insert_queries( 140 self, table: str, columns: tuple[str, ...], values: list[tuple[Any, ...]] 141 ) -> list[tuple[str, tuple[Any, ...]]]: 142 """Format a list of insert statements. 143 144 The returned value is a list because `values` will be broken up into chunks. 145 146 Each list element is a two tuple consisting of the parameterized query string and a tuple of values.""" 147 inserts = [] 148 max_row_count = 900 149 column_list = "(" + ", ".join(columns) + ")" 150 for i in range(0, len(values), max_row_count): 151 chunk = values[i : i + max_row_count] 152 placeholder = ( 153 "(" + "),(".join((", ".join(("?" for _ in row)) for row in chunk)) + ")" 154 ) 155 flattened_values = tuple((value for row in chunk for value in row)) 156 inserts.append( 157 ( 158 f"INSERT INTO {table} {column_list} VALUES {placeholder};", 159 flattened_values, 160 ) 161 ) 162 return inserts 163 164 def _set_foreign_key_enforcement(self): 165 if self.connection: 166 self.connection.execute( 167 f"pragma foreign_keys = {int(self.enforce_foreign_keys)};" 168 ) 169 170 def add_column(self, table: str, column_def: str): 171 """Add a column to `table`. 172 173 `column_def` should be in the form `{column_name} {type_name} {constraint}`. 174 175 i.e. 176 >>> db = Databased() 177 >>> db.add_column("rides", "num_stops INTEGER NOT NULL DEFAULT 0")""" 178 self.query(f"ALTER TABLE {table} ADD {column_def};") 179 180 def close(self): 181 """Disconnect from the database. 182 183 Does not call `commit()` for you unless the `commit_on_close` property is set to `True`. 184 """ 185 if self.connection: 186 if self.commit_on_close: 187 self.commit() 188 self.connection.close() 189 self.connection = None 190 191 def commit(self): 192 """Commit state of database.""" 193 if self.connection: 194 self.connection.commit() 195 self.logger.info("Committed successfully.") 196 else: 197 raise RuntimeError( 198 "Databased.commit(): Can't commit db with no open connection." 199 ) 200 201 def connect(self): 202 """Connect to the database.""" 203 self.connection = sqlite3.connect( 204 self.path, 205 detect_types=sqlite3.PARSE_DECLTYPES | sqlite3.PARSE_COLNAMES 206 if self.detect_types 207 else 0, 208 timeout=self.connection_timeout, 209 ) 210 self._set_foreign_key_enforcement() 211 self.connection.row_factory = dict_factory 212 213 def count( 214 self, 215 table: str, 216 column: str = "*", 217 where: str | None = None, 218 distinct: bool = False, 219 ) -> int: 220 """Return number of matching rows in `table` table. 221 222 Equivalent to: 223 >>> SELECT COUNT({distinct} {column}) FROM {table} {where};""" 224 query = ( 225 f"SELECT COUNT( {('DISTINCT' if distinct else '')} {column}) FROM {table}" 226 ) 227 if where: 228 query += f" WHERE {where}" 229 query += ";" 230 return int(list(self.query(query)[0].values())[0]) 231 232 def create_table(self, table: str, *column_defs: str): 233 """Create a table if it doesn't exist. 234 235 #### :params: 236 237 `table`: Name of the table to create. 238 239 `column_defs`: Any number of column names and their definitions in proper Sqlite3 sytax. 240 i.e. `"column_name TEXT UNIQUE"` or `"column_name INTEGER PRIMARY KEY"` etc.""" 241 columns = ", ".join(column_defs) 242 result = self.query(f"CREATE TABLE IF NOT EXISTS {table} ({columns});") 243 self.logger.info(f"'{table}' table created.") 244 245 def delete(self, table: str, where: str | None = None) -> int: 246 """Delete rows from `table` that satisfy the given `where` clause. 247 248 If `where` is `None`, all rows will be deleted. 249 250 Returns the number of deleted rows. 251 252 e.g. 253 >>> db = Databased() 254 >>> db.delete("rides", "distance < 5 AND average_speed < 7")""" 255 try: 256 if where: 257 self.query(f"DELETE FROM {table} WHERE {where};") 258 else: 259 self.query(f"DELETE FROM {table};") 260 row_count = self.cursor.rowcount 261 self.logger.info( 262 f"Deleted {row_count} rows from '{table}' where '{where}'." 263 ) 264 return row_count 265 except Exception as e: 266 self.logger.exception( 267 f"Error deleting rows from '{table}' where '{where}'." 268 ) 269 raise e 270 271 def describe(self, table: str) -> list[dict]: 272 """Returns information about `table`.""" 273 return self.query(f"pragma table_info('{table}');") 274 275 def drop_column(self, table: str, column: str): 276 """Drop `column` from `table`.""" 277 self.query(f"ALTER TABLE {table} DROP {column};") 278 279 def drop_table(self, table: str) -> bool: 280 """Drop `table` from the database. 281 282 Returns `True` if successful, `False` if not.""" 283 try: 284 self.query(f"DROP TABLE {table};") 285 self.logger.info(f"Dropped table '{table}'.") 286 return True 287 except Exception as e: 288 print(f"{type(e).__name__}: {e}") 289 self.logger.error(f"Failed to drop table '{table}'.") 290 return False 291 292 def execute_script(self, path: Pathish, encoding: str = "utf-8") -> list[dict]: 293 """Execute sql script located at `path`.""" 294 if not self.connected: 295 self.connect() 296 assert self.connection 297 return self.connection.executescript( 298 Pathier(path).read_text(encoding) 299 ).fetchall() 300 301 def get_columns(self, table: str) -> tuple[str, ...]: 302 """Returns a list of column names in `table`.""" 303 return tuple( 304 (column["name"] for column in self.query(f"pragma table_info('{table}');")) 305 ) 306 307 def insert( 308 self, table: str, columns: tuple[str, ...], values: list[tuple[Any, ...]] 309 ) -> int: 310 """Insert rows of `values` into `columns` of `table`. 311 312 Each `tuple` in `values` corresponds to an individual row that is to be inserted. 313 """ 314 row_count = 0 315 for insert in self._prepare_insert_queries(table, columns, values): 316 try: 317 self.query(insert[0], insert[1]) 318 row_count += self.cursor.rowcount 319 self.logger.info(f"Inserted {row_count} rows into '{table}' table.") 320 except Exception as e: 321 self.logger.exception(f"Error inserting rows into '{table}' table.") 322 raise e 323 return row_count 324 325 def query(self, query_: str, parameters: tuple[Any, ...] = tuple()) -> list[dict]: 326 """Execute an SQL query and return the results. 327 328 Ensures that the database connection is opened before executing the command. 329 330 The cursor used to execute the query will be available through `self.cursor` until the next time `self.query()` is called. 331 """ 332 if not self.connected: 333 self.connect() 334 assert self.connection 335 self.cursor = self.connection.cursor() 336 self.cursor.execute(query_, parameters) 337 return self.cursor.fetchall() 338 339 def rename_column(self, table: str, column_to_rename: str, new_column_name: str): 340 """Rename a column in `table`.""" 341 self.query( 342 f"ALTER TABLE {table} RENAME {column_to_rename} TO {new_column_name};" 343 ) 344 345 def rename_table(self, table_to_rename: str, new_table_name: str): 346 """Rename a table.""" 347 self.query(f"ALTER TABLE {table_to_rename} RENAME TO {new_table_name};") 348 349 def select( 350 self, 351 table: str, 352 columns: list[str] = ["*"], 353 joins: list[str] | None = None, 354 where: str | None = None, 355 group_by: str | None = None, 356 having: str | None = None, 357 order_by: str | None = None, 358 limit: int | str | None = None, 359 ) -> list[dict]: 360 """Return rows for given criteria. 361 362 For complex queries, use the `databased.query()` method. 363 364 Parameters `where`, `group_by`, `having`, `order_by`, and `limit` should not have 365 their corresponding key word in their string, but should otherwise be valid SQL. 366 367 `joins` should contain their key word (`INNER JOIN`, `LEFT JOIN`) in addition to the rest of the sub-statement. 368 369 >>> Databased().select( 370 "bike_rides", 371 "id, date, distance, moving_time, AVG(distance/moving_time) as average_speed", 372 where="distance > 20", 373 order_by="distance", 374 desc=True, 375 limit=10 376 ) 377 executes the query: 378 >>> SELECT 379 id, date, distance, moving_time, AVG(distance/moving_time) as average_speed 380 FROM 381 bike_rides 382 WHERE 383 distance > 20 384 ORDER BY 385 distance DESC 386 Limit 10;""" 387 query = f"SELECT {', '.join(columns)} FROM {table}" 388 if joins: 389 query += f" {' '.join(joins)}" 390 if where: 391 query += f" WHERE {where}" 392 if group_by: 393 query += f" GROUP BY {group_by}" 394 if having: 395 query += f" HAVING {having}" 396 if order_by: 397 query += f" ORDER BY {order_by}" 398 if limit: 399 query += f" LIMIT {limit}" 400 query += ";" 401 rows = self.query(query) 402 return rows 403 404 @staticmethod 405 def to_grid(data: list[dict], shrink_to_terminal: bool = True) -> str: 406 """Returns a tabular grid from `data`. 407 408 If `shrink_to_terminal` is `True`, the column widths of the grid will be reduced to fit within the current terminal. 409 """ 410 return griddy(data, "keys", shrink_to_terminal) 411 412 def update( 413 self, table: str, column: str, value: Any, where: str | None = None 414 ) -> int: 415 """Update `column` of `table` to `value` for rows satisfying the conditions in `where`. 416 417 If `where` is `None` all rows will be updated. 418 419 Returns the number of updated rows. 420 421 e.g. 422 >>> db = Databased() 423 >>> db.update("rides", "elevation", 100, "elevation < 100")""" 424 try: 425 if where: 426 self.query(f"UPDATE {table} SET {column} = ? WHERE {where};", (value,)) 427 else: 428 self.query(f"UPDATE {table} SET {column} = ?;", (value,)) 429 row_count = self.cursor.rowcount 430 self.logger.info( 431 f"Updated {row_count} rows in '{table}' table to '{column}' = '{value}' where '{where}'." 432 ) 433 return row_count 434 except Exception as e: 435 self.logger.exception( 436 f"Failed to update rows in '{table}' table to '{column}' = '{value}' where '{where}'." 437 ) 438 raise e 439 440 def vacuum(self) -> int: 441 """Reduce disk size of database after row/table deletion. 442 443 Returns space freed up in bytes.""" 444 size = self.path.size 445 self.query("VACUUM;") 446 return size - self.path.size 447 448 # Seat ========================== Database Dump ========================================= 449 450 def _format_column_def(self, description: dict) -> str: 451 name = description["name"] 452 type_ = description["type"] 453 primary_key = bool(description["pk"]) 454 not_null = bool(description["notnull"]) 455 default = description["dflt_value"] 456 column = f"{name} {type_}" 457 if primary_key: 458 column += f" PRIMARY KEY" 459 if not_null: 460 column += f" NOT NULL" 461 if default: 462 if isinstance(default, str): 463 default = f"{default}" 464 column += f" DEFAULT {default}" 465 return column 466 467 def _format_table_data(self, table: str) -> str: 468 columns = self.get_columns(table) 469 rows = [tuple(row.values()) for row in self.select(table)] 470 inserts = self._prepare_insert_queries(table, columns, rows) 471 insert_strings = [] 472 indent = " " * 4 473 for insert in inserts: 474 text = insert[0] 475 sub = "^$data$based$^" 476 text = text.replace("?", sub) 477 for value in insert[1]: 478 if not value: 479 value = "" 480 if isinstance(value, bool): 481 value = int(value) 482 if not isinstance(value, int) and (not isinstance(value, float)): 483 if isinstance(value, str): 484 value = value.replace('"', "'") 485 value = f'"{value}"' 486 text = text.replace(sub, str(value), 1) 487 for pair in [ 488 ("INSERT INTO ", f"INSERT INTO\n{indent}"), 489 (") VALUES (", f")\nVALUES\n{indent}("), 490 ("),", f"),\n{indent}"), 491 ]: 492 text = text.replace(pair[0], pair[1]) 493 insert_strings.append(text) 494 return "\n".join(insert_strings) 495 496 def _format_table_def(self, table: str) -> str: 497 description = self.describe(table) 498 indent = " " * 4 499 columns = ",\n".join( 500 (f"{indent * 2}{self._format_column_def(column)}" for column in description) 501 ) 502 table_def = ( 503 "CREATE TABLE IF NOT EXISTS\n" 504 + f"{indent}{table} (\n" 505 + columns 506 + f"\n{indent});" 507 ) 508 return table_def 509 510 def _get_data_dump_string(self, tables: list[str]) -> str: 511 return "\n\n".join((self._format_table_data(table) for table in tables)) 512 513 def _get_schema_dump_string(self, tables: list[str]) -> str: 514 return "\n\n".join((self._format_table_def(table) for table in tables)) 515 516 def dump_data(self, path: Pathish, tables: list[str] | None = None): 517 """Create a data dump file for the specified tables or all tables, if none are given.""" 518 tables = tables or self.tables 519 path = Pathier(path) 520 path.write_text(self._get_data_dump_string(tables), encoding="utf-8") 521 522 def dump_schema(self, path: Pathish, tables: list[str] | None = None): 523 """Create a schema dump file for the specified tables or all tables, if none are given. 524 525 NOTE: Foreign key relationships/constraints are not preserved when dumping the schema.""" 526 tables = tables or self.tables 527 path = Pathier(path) 528 path.write_text(self._get_schema_dump_string(tables), encoding="utf-8")
SQLite3 wrapper.
18 def __init__( 19 self, 20 dbpath: Pathish = "db.sqlite3", 21 connection_timeout: float = 10, 22 detect_types: bool = True, 23 enforce_foreign_keys: bool = True, 24 commit_on_close: bool = True, 25 logger_encoding: str = "utf-8", 26 logger_message_format: str = "{levelname}|-|{asctime}|-|{message}", 27 ): 28 """ """ 29 self.path = dbpath 30 self.connection_timeout = connection_timeout 31 self.connection = None 32 self._logger_init(logger_message_format, logger_encoding) 33 self.detect_types = detect_types 34 self.commit_on_close = commit_on_close 35 self.enforce_foreign_keys = enforce_foreign_keys
If new_path
doesn't exist, it will be created (including parent folders).
Changes to this property won't take effect until the current connection, if open, is closed and a new connection opened.
Should use detect_types=sqlite3.PARSE_DECLTYPES | sqlite3.PARSE_COLNAMES
when establishing a database connection.
Changes to this property won't take effect until the current connection, if open, is closed and a new connection opened.
170 def add_column(self, table: str, column_def: str): 171 """Add a column to `table`. 172 173 `column_def` should be in the form `{column_name} {type_name} {constraint}`. 174 175 i.e. 176 >>> db = Databased() 177 >>> db.add_column("rides", "num_stops INTEGER NOT NULL DEFAULT 0")""" 178 self.query(f"ALTER TABLE {table} ADD {column_def};")
Add a column to table
.
column_def
should be in the form {column_name} {type_name} {constraint}
.
i.e.
>>> db = Databased()
>>> db.add_column("rides", "num_stops INTEGER NOT NULL DEFAULT 0")
180 def close(self): 181 """Disconnect from the database. 182 183 Does not call `commit()` for you unless the `commit_on_close` property is set to `True`. 184 """ 185 if self.connection: 186 if self.commit_on_close: 187 self.commit() 188 self.connection.close() 189 self.connection = None
Disconnect from the database.
Does not call commit()
for you unless the commit_on_close
property is set to True
.
191 def commit(self): 192 """Commit state of database.""" 193 if self.connection: 194 self.connection.commit() 195 self.logger.info("Committed successfully.") 196 else: 197 raise RuntimeError( 198 "Databased.commit(): Can't commit db with no open connection." 199 )
Commit state of database.
201 def connect(self): 202 """Connect to the database.""" 203 self.connection = sqlite3.connect( 204 self.path, 205 detect_types=sqlite3.PARSE_DECLTYPES | sqlite3.PARSE_COLNAMES 206 if self.detect_types 207 else 0, 208 timeout=self.connection_timeout, 209 ) 210 self._set_foreign_key_enforcement() 211 self.connection.row_factory = dict_factory
Connect to the database.
213 def count( 214 self, 215 table: str, 216 column: str = "*", 217 where: str | None = None, 218 distinct: bool = False, 219 ) -> int: 220 """Return number of matching rows in `table` table. 221 222 Equivalent to: 223 >>> SELECT COUNT({distinct} {column}) FROM {table} {where};""" 224 query = ( 225 f"SELECT COUNT( {('DISTINCT' if distinct else '')} {column}) FROM {table}" 226 ) 227 if where: 228 query += f" WHERE {where}" 229 query += ";" 230 return int(list(self.query(query)[0].values())[0])
Return number of matching rows in table
table.
Equivalent to:
>>> SELECT COUNT({distinct} {column}) FROM {table} {where};
232 def create_table(self, table: str, *column_defs: str): 233 """Create a table if it doesn't exist. 234 235 #### :params: 236 237 `table`: Name of the table to create. 238 239 `column_defs`: Any number of column names and their definitions in proper Sqlite3 sytax. 240 i.e. `"column_name TEXT UNIQUE"` or `"column_name INTEGER PRIMARY KEY"` etc.""" 241 columns = ", ".join(column_defs) 242 result = self.query(f"CREATE TABLE IF NOT EXISTS {table} ({columns});") 243 self.logger.info(f"'{table}' table created.")
Create a table if it doesn't exist.
:params:
table
: Name of the table to create.
column_defs
: Any number of column names and their definitions in proper Sqlite3 sytax.
i.e. "column_name TEXT UNIQUE"
or "column_name INTEGER PRIMARY KEY"
etc.
245 def delete(self, table: str, where: str | None = None) -> int: 246 """Delete rows from `table` that satisfy the given `where` clause. 247 248 If `where` is `None`, all rows will be deleted. 249 250 Returns the number of deleted rows. 251 252 e.g. 253 >>> db = Databased() 254 >>> db.delete("rides", "distance < 5 AND average_speed < 7")""" 255 try: 256 if where: 257 self.query(f"DELETE FROM {table} WHERE {where};") 258 else: 259 self.query(f"DELETE FROM {table};") 260 row_count = self.cursor.rowcount 261 self.logger.info( 262 f"Deleted {row_count} rows from '{table}' where '{where}'." 263 ) 264 return row_count 265 except Exception as e: 266 self.logger.exception( 267 f"Error deleting rows from '{table}' where '{where}'." 268 ) 269 raise e
Delete rows from table
that satisfy the given where
clause.
If where
is None
, all rows will be deleted.
Returns the number of deleted rows.
e.g.
>>> db = Databased()
>>> db.delete("rides", "distance < 5 AND average_speed < 7")
271 def describe(self, table: str) -> list[dict]: 272 """Returns information about `table`.""" 273 return self.query(f"pragma table_info('{table}');")
Returns information about table
.
275 def drop_column(self, table: str, column: str): 276 """Drop `column` from `table`.""" 277 self.query(f"ALTER TABLE {table} DROP {column};")
Drop column
from table
.
279 def drop_table(self, table: str) -> bool: 280 """Drop `table` from the database. 281 282 Returns `True` if successful, `False` if not.""" 283 try: 284 self.query(f"DROP TABLE {table};") 285 self.logger.info(f"Dropped table '{table}'.") 286 return True 287 except Exception as e: 288 print(f"{type(e).__name__}: {e}") 289 self.logger.error(f"Failed to drop table '{table}'.") 290 return False
Drop table
from the database.
Returns True
if successful, False
if not.
292 def execute_script(self, path: Pathish, encoding: str = "utf-8") -> list[dict]: 293 """Execute sql script located at `path`.""" 294 if not self.connected: 295 self.connect() 296 assert self.connection 297 return self.connection.executescript( 298 Pathier(path).read_text(encoding) 299 ).fetchall()
Execute sql script located at path
.
301 def get_columns(self, table: str) -> tuple[str, ...]: 302 """Returns a list of column names in `table`.""" 303 return tuple( 304 (column["name"] for column in self.query(f"pragma table_info('{table}');")) 305 )
Returns a list of column names in table
.
307 def insert( 308 self, table: str, columns: tuple[str, ...], values: list[tuple[Any, ...]] 309 ) -> int: 310 """Insert rows of `values` into `columns` of `table`. 311 312 Each `tuple` in `values` corresponds to an individual row that is to be inserted. 313 """ 314 row_count = 0 315 for insert in self._prepare_insert_queries(table, columns, values): 316 try: 317 self.query(insert[0], insert[1]) 318 row_count += self.cursor.rowcount 319 self.logger.info(f"Inserted {row_count} rows into '{table}' table.") 320 except Exception as e: 321 self.logger.exception(f"Error inserting rows into '{table}' table.") 322 raise e 323 return row_count
Insert rows of values
into columns
of table
.
Each tuple
in values
corresponds to an individual row that is to be inserted.
325 def query(self, query_: str, parameters: tuple[Any, ...] = tuple()) -> list[dict]: 326 """Execute an SQL query and return the results. 327 328 Ensures that the database connection is opened before executing the command. 329 330 The cursor used to execute the query will be available through `self.cursor` until the next time `self.query()` is called. 331 """ 332 if not self.connected: 333 self.connect() 334 assert self.connection 335 self.cursor = self.connection.cursor() 336 self.cursor.execute(query_, parameters) 337 return self.cursor.fetchall()
Execute an SQL query and return the results.
Ensures that the database connection is opened before executing the command.
The cursor used to execute the query will be available through self.cursor
until the next time self.query()
is called.
339 def rename_column(self, table: str, column_to_rename: str, new_column_name: str): 340 """Rename a column in `table`.""" 341 self.query( 342 f"ALTER TABLE {table} RENAME {column_to_rename} TO {new_column_name};" 343 )
Rename a column in table
.
345 def rename_table(self, table_to_rename: str, new_table_name: str): 346 """Rename a table.""" 347 self.query(f"ALTER TABLE {table_to_rename} RENAME TO {new_table_name};")
Rename a table.
349 def select( 350 self, 351 table: str, 352 columns: list[str] = ["*"], 353 joins: list[str] | None = None, 354 where: str | None = None, 355 group_by: str | None = None, 356 having: str | None = None, 357 order_by: str | None = None, 358 limit: int | str | None = None, 359 ) -> list[dict]: 360 """Return rows for given criteria. 361 362 For complex queries, use the `databased.query()` method. 363 364 Parameters `where`, `group_by`, `having`, `order_by`, and `limit` should not have 365 their corresponding key word in their string, but should otherwise be valid SQL. 366 367 `joins` should contain their key word (`INNER JOIN`, `LEFT JOIN`) in addition to the rest of the sub-statement. 368 369 >>> Databased().select( 370 "bike_rides", 371 "id, date, distance, moving_time, AVG(distance/moving_time) as average_speed", 372 where="distance > 20", 373 order_by="distance", 374 desc=True, 375 limit=10 376 ) 377 executes the query: 378 >>> SELECT 379 id, date, distance, moving_time, AVG(distance/moving_time) as average_speed 380 FROM 381 bike_rides 382 WHERE 383 distance > 20 384 ORDER BY 385 distance DESC 386 Limit 10;""" 387 query = f"SELECT {', '.join(columns)} FROM {table}" 388 if joins: 389 query += f" {' '.join(joins)}" 390 if where: 391 query += f" WHERE {where}" 392 if group_by: 393 query += f" GROUP BY {group_by}" 394 if having: 395 query += f" HAVING {having}" 396 if order_by: 397 query += f" ORDER BY {order_by}" 398 if limit: 399 query += f" LIMIT {limit}" 400 query += ";" 401 rows = self.query(query) 402 return rows
Return rows for given criteria.
For complex queries, use the databased.query()
method.
Parameters where
, group_by
, having
, order_by
, and limit
should not have
their corresponding key word in their string, but should otherwise be valid SQL.
joins
should contain their key word (INNER JOIN
, LEFT JOIN
) in addition to the rest of the sub-statement.
>>> Databased().select(
"bike_rides",
"id, date, distance, moving_time, AVG(distance/moving_time) as average_speed",
where="distance > 20",
order_by="distance",
desc=True,
limit=10
)
executes the query:
>>> SELECT
id, date, distance, moving_time, AVG(distance/moving_time) as average_speed
FROM
bike_rides
WHERE
distance > 20
ORDER BY
distance DESC
Limit 10;
404 @staticmethod 405 def to_grid(data: list[dict], shrink_to_terminal: bool = True) -> str: 406 """Returns a tabular grid from `data`. 407 408 If `shrink_to_terminal` is `True`, the column widths of the grid will be reduced to fit within the current terminal. 409 """ 410 return griddy(data, "keys", shrink_to_terminal)
Returns a tabular grid from data
.
If shrink_to_terminal
is True
, the column widths of the grid will be reduced to fit within the current terminal.
412 def update( 413 self, table: str, column: str, value: Any, where: str | None = None 414 ) -> int: 415 """Update `column` of `table` to `value` for rows satisfying the conditions in `where`. 416 417 If `where` is `None` all rows will be updated. 418 419 Returns the number of updated rows. 420 421 e.g. 422 >>> db = Databased() 423 >>> db.update("rides", "elevation", 100, "elevation < 100")""" 424 try: 425 if where: 426 self.query(f"UPDATE {table} SET {column} = ? WHERE {where};", (value,)) 427 else: 428 self.query(f"UPDATE {table} SET {column} = ?;", (value,)) 429 row_count = self.cursor.rowcount 430 self.logger.info( 431 f"Updated {row_count} rows in '{table}' table to '{column}' = '{value}' where '{where}'." 432 ) 433 return row_count 434 except Exception as e: 435 self.logger.exception( 436 f"Failed to update rows in '{table}' table to '{column}' = '{value}' where '{where}'." 437 ) 438 raise e
Update column
of table
to value
for rows satisfying the conditions in where
.
If where
is None
all rows will be updated.
Returns the number of updated rows.
e.g.
>>> db = Databased()
>>> db.update("rides", "elevation", 100, "elevation < 100")
440 def vacuum(self) -> int: 441 """Reduce disk size of database after row/table deletion. 442 443 Returns space freed up in bytes.""" 444 size = self.path.size 445 self.query("VACUUM;") 446 return size - self.path.size
Reduce disk size of database after row/table deletion.
Returns space freed up in bytes.
516 def dump_data(self, path: Pathish, tables: list[str] | None = None): 517 """Create a data dump file for the specified tables or all tables, if none are given.""" 518 tables = tables or self.tables 519 path = Pathier(path) 520 path.write_text(self._get_data_dump_string(tables), encoding="utf-8")
Create a data dump file for the specified tables or all tables, if none are given.
522 def dump_schema(self, path: Pathish, tables: list[str] | None = None): 523 """Create a schema dump file for the specified tables or all tables, if none are given. 524 525 NOTE: Foreign key relationships/constraints are not preserved when dumping the schema.""" 526 tables = tables or self.tables 527 path = Pathier(path) 528 path.write_text(self._get_schema_dump_string(tables), encoding="utf-8")
Create a schema dump file for the specified tables or all tables, if none are given.
NOTE: Foreign key relationships/constraints are not preserved when dumping the schema.