databased.databased
1import logging 2import sqlite3 3from typing import Any 4 5from griddle import griddy 6from pathier import Pathier, Pathish 7 8 9def dict_factory(cursor: sqlite3.Cursor, row: tuple) -> dict: 10 fields = [column[0] for column in cursor.description] 11 return {column: value for column, value in zip(fields, row)} 12 13 14class Databased: 15 """SQLite3 wrapper.""" 16 17 def __init__( 18 self, 19 dbpath: Pathish = "db.sqlite3", 20 connection_timeout: float = 10, 21 detect_types: bool = True, 22 enforce_foreign_keys: bool = True, 23 commit_on_close: bool = True, 24 logger_encoding: str = "utf-8", 25 logger_message_format: str = "{levelname}|-|{asctime}|-|{message}", 26 ): 27 """ """ 28 self.path = dbpath 29 self.connection_timeout = connection_timeout 30 self.connection = None 31 self._logger_init(logger_message_format, logger_encoding) 32 self.detect_types = detect_types 33 self.commit_on_close = commit_on_close 34 self.enforce_foreign_keys = enforce_foreign_keys 35 36 def __enter__(self): 37 self.connect() 38 return self 39 40 def __exit__(self, *args, **kwargs): 41 self.close() 42 43 @property 44 def commit_on_close(self) -> bool: 45 """Should commit database before closing connection when `self.close()` is called.""" 46 return self._commit_on_close 47 48 @commit_on_close.setter 49 def commit_on_close(self, should_commit_on_close: bool): 50 self._commit_on_close = should_commit_on_close 51 52 @property 53 def connected(self) -> bool: 54 """Whether this `Databased` instance is connected to the database file or not.""" 55 return self.connection is not None 56 57 @property 58 def connection_timeout(self) -> float: 59 """Changes to this property won't take effect until the current connection, if open, is closed and a new connection opened.""" 60 return self._connection_timeout 61 62 @connection_timeout.setter 63 def connection_timeout(self, timeout: float): 64 self._connection_timeout = timeout 65 66 @property 67 def detect_types(self) -> bool: 68 """Should use `detect_types=sqlite3.PARSE_DECLTYPES | sqlite3.PARSE_COLNAMES` when establishing a database connection. 69 70 Changes to this property won't take effect until the current connection, if open, is closed and a new connection opened. 71 """ 72 return self._detect_types 73 74 @detect_types.setter 75 def detect_types(self, should_detect: bool): 76 self._detect_types = should_detect 77 78 @property 79 def enforce_foreign_keys(self) -> bool: 80 return self._enforce_foreign_keys 81 82 @enforce_foreign_keys.setter 83 def enforce_foreign_keys(self, should_enforce: bool): 84 self._enforce_foreign_keys = should_enforce 85 self._set_foreign_key_enforcement() 86 87 @property 88 def indicies(self) -> list[str]: 89 """List of indicies for this database.""" 90 return [ 91 table["name"] 92 for table in self.query( 93 "SELECT name FROM sqlite_Schema WHERE type = 'index';" 94 ) 95 ] 96 97 @property 98 def name(self) -> str: 99 """The name of this database.""" 100 return self.path.stem 101 102 @property 103 def path(self) -> Pathier: 104 """The path to this database file.""" 105 return self._path 106 107 @path.setter 108 def path(self, new_path: Pathish): 109 """If `new_path` doesn't exist, it will be created (including parent folders).""" 110 self._path = Pathier(new_path) 111 if not self.path.exists(): 112 self.path.touch() 113 114 @property 115 def tables(self) -> list[str]: 116 """List of table names for this database.""" 117 return [ 118 table["name"] 119 for table in self.query( 120 "SELECT name FROM sqlite_Schema WHERE type = 'table' AND name NOT LIKE 'sqlite_%';" 121 ) 122 ] 123 124 @property 125 def views(self) -> list[str]: 126 """List of view for this database.""" 127 return [ 128 table["name"] 129 for table in self.query( 130 "SELECT name FROM sqlite_Schema WHERE type = 'view' AND name NOT LIKE 'sqlite_%';" 131 ) 132 ] 133 134 def _logger_init(self, message_format: str, encoding: str): 135 """:param: `message_format`: `{` style format string.""" 136 self.logger = logging.getLogger(self.name) 137 if not self.logger.hasHandlers(): 138 handler = logging.FileHandler( 139 str(self.path).replace(".", "") + ".log", encoding=encoding 140 ) 141 handler.setFormatter( 142 logging.Formatter( 143 message_format, style="{", datefmt="%m/%d/%Y %I:%M:%S %p" 144 ) 145 ) 146 self.logger.addHandler(handler) 147 self.logger.setLevel(logging.INFO) 148 149 def _set_foreign_key_enforcement(self): 150 if self.connection: 151 self.connection.execute( 152 f"pragma foreign_keys = {int(self.enforce_foreign_keys)};" 153 ) 154 155 def add_column(self, table: str, column_def: str): 156 """Add a column to `table`. 157 158 `column_def` should be in the form `{column_name} {type_name} {constraint}`. 159 160 i.e. 161 >>> db = Databased() 162 >>> db.add_column("rides", "num_stops INTEGER NOT NULL DEFAULT 0")""" 163 self.query(f"ALTER TABLE {table} ADD {column_def};") 164 165 def close(self): 166 """Disconnect from the database. 167 168 Does not call `commit()` for you unless the `commit_on_close` property is set to `True`. 169 """ 170 if self.connection: 171 if self.commit_on_close: 172 self.commit() 173 self.connection.close() 174 self.connection = None 175 176 def commit(self): 177 """Commit state of database.""" 178 if self.connection: 179 self.connection.commit() 180 self.logger.info("Committed successfully.") 181 else: 182 raise RuntimeError( 183 "Databased.commit(): Can't commit db with no open connection." 184 ) 185 186 def connect(self): 187 """Connect to the database.""" 188 self.connection = sqlite3.connect( 189 self.path, 190 detect_types=sqlite3.PARSE_DECLTYPES | sqlite3.PARSE_COLNAMES 191 if self.detect_types 192 else 0, 193 timeout=self.connection_timeout, 194 ) 195 self._set_foreign_key_enforcement() 196 self.connection.row_factory = dict_factory 197 198 def count( 199 self, 200 table: str, 201 column: str = "*", 202 where: str | None = None, 203 distinct: bool = False, 204 ) -> int: 205 """Return number of matching rows in `table` table. 206 207 Equivalent to: 208 >>> SELECT COUNT({distinct} {column}) FROM {table} {where};""" 209 query = ( 210 f"SELECT COUNT( {('DISTINCT' if distinct else '')} {column}) FROM {table}" 211 ) 212 if where: 213 query += f" WHERE {where}" 214 query += ";" 215 return int(list(self.query(query)[0].values())[0]) 216 217 def create_table(self, table: str, *column_defs: str): 218 """Create a table if it doesn't exist. 219 220 #### :params: 221 222 `table`: Name of the table to create. 223 224 `column_defs`: Any number of column names and their definitions in proper Sqlite3 sytax. 225 i.e. `"column_name TEXT UNIQUE"` or `"column_name INTEGER PRIMARY KEY"` etc.""" 226 columns = ", ".join(column_defs) 227 result = self.query(f"CREATE TABLE IF NOT EXISTS {table} ({columns});") 228 self.logger.info(f"'{table}' table created.") 229 230 def delete(self, table: str, where: str | None = None) -> int: 231 """Delete rows from `table` that satisfy the given `where` clause. 232 233 If `where` is `None`, all rows will be deleted. 234 235 Returns the number of deleted rows. 236 237 e.g. 238 >>> db = Databased() 239 >>> db.delete("rides", "distance < 5 AND average_speed < 7")""" 240 try: 241 if where: 242 self.query(f"DELETE FROM {table} WHERE {where};") 243 else: 244 self.query(f"DELETE FROM {table};") 245 row_count = self.cursor.rowcount 246 self.logger.info( 247 f"Deleted {row_count} rows from '{table}' where '{where}'." 248 ) 249 return row_count 250 except Exception as e: 251 self.logger.exception( 252 f"Error deleting rows from '{table}' where '{where}'." 253 ) 254 raise e 255 256 def describe(self, table: str) -> list[dict]: 257 """Returns information about `table`.""" 258 return self.query(f"pragma table_info('{table}');") 259 260 def drop_column(self, table: str, column: str): 261 """Drop `column` from `table`.""" 262 self.query(f"ALTER TABLE {table} DROP {column};") 263 264 def drop_table(self, table: str) -> bool: 265 """Drop `table` from the database. 266 267 Returns `True` if successful, `False` if not.""" 268 try: 269 self.query(f"DROP TABLE {table};") 270 self.logger.info(f"Dropped table '{table}'.") 271 return True 272 except Exception as e: 273 print(f"{type(e).__name__}: {e}") 274 self.logger.error(f"Failed to drop table '{table}'.") 275 return False 276 277 def execute_script(self, path: Pathish, encoding: str = "utf-8") -> list[dict]: 278 """Execute sql script located at `path`.""" 279 if not self.connected: 280 self.connect() 281 assert self.connection 282 script = Pathier(path).read_text(encoding).replace("\n", " ") 283 return self.query(script) 284 285 def get_columns(self, table: str) -> tuple[str, ...]: 286 """Returns a list of column names in `table`.""" 287 return tuple( 288 (column["name"] for column in self.query(f"pragma table_info('{table}');")) 289 ) 290 291 def insert( 292 self, table: str, columns: tuple[str, ...], values: list[tuple[Any, ...]] 293 ) -> int: 294 """Insert rows of `values` into `columns` of `table`. 295 296 Each `tuple` in `values` corresponds to an individual row that is to be inserted. 297 """ 298 max_row_count = 900 299 column_list = "(" + ", ".join(columns) + ")" 300 row_count = 0 301 for i in range(0, len(values), max_row_count): 302 chunk = values[i : i + max_row_count] 303 placeholder = ( 304 "(" + "),(".join((", ".join(("?" for _ in row)) for row in chunk)) + ")" 305 ) 306 logger_values = "\n".join( 307 ( 308 "'(" + ", ".join((str(value) for value in row)) + ")'" 309 for row in chunk 310 ) 311 ) 312 flattened_values = tuple((value for row in chunk for value in row)) 313 try: 314 self.query( 315 f"INSERT INTO {table} {column_list} VALUES {placeholder};", 316 flattened_values, 317 ) 318 self.logger.info( 319 f"Inserted into '{column_list}' columns of '{table}' table values \n{logger_values}." 320 ) 321 row_count += self.cursor.rowcount 322 except Exception as e: 323 self.logger.exception( 324 f"Error inserting into '{column_list}' columns of '{table}' table values \n{logger_values}." 325 ) 326 raise e 327 return row_count 328 329 def query(self, query_: str, parameters: tuple[Any, ...] = tuple()) -> list[dict]: 330 """Execute an SQL query and return the results. 331 332 Ensures that the database connection is opened before executing the command. 333 334 The cursor used to execute the query will be available through `self.cursor` until the next time `self.query()` is called. 335 """ 336 if not self.connected: 337 self.connect() 338 assert self.connection 339 self.cursor = self.connection.cursor() 340 self.cursor.execute(query_, parameters) 341 return self.cursor.fetchall() 342 343 def rename_column(self, table: str, column_to_rename: str, new_column_name: str): 344 """Rename a column in `table`.""" 345 self.query( 346 f"ALTER TABLE {table} RENAME {column_to_rename} TO {new_column_name};" 347 ) 348 349 def rename_table(self, table_to_rename: str, new_table_name: str): 350 """Rename a table.""" 351 self.query(f"ALTER TABLE {table_to_rename} RENAME TO {new_table_name};") 352 353 def select( 354 self, 355 table: str, 356 columns: list[str] = ["*"], 357 joins: list[str] | None = None, 358 where: str | None = None, 359 group_by: str | None = None, 360 having: str | None = None, 361 order_by: str | None = None, 362 limit: int | str | None = None, 363 ) -> list[dict]: 364 """Return rows for given criteria. 365 366 For complex queries, use the `databased.query()` method. 367 368 Parameters `where`, `group_by`, `having`, `order_by`, and `limit` should not have 369 their corresponding key word in their string, but should otherwise be valid SQL. 370 371 `joins` should contain their key word (`INNER JOIN`, `LEFT JOIN`) in addition to the rest of the sub-statement. 372 373 >>> Databased().select( 374 "bike_rides", 375 "id, date, distance, moving_time, AVG(distance/moving_time) as average_speed", 376 where="distance > 20", 377 order_by="distance", 378 desc=True, 379 limit=10 380 ) 381 executes the query: 382 >>> SELECT 383 id, date, distance, moving_time, AVG(distance/moving_time) as average_speed 384 FROM 385 bike_rides 386 WHERE 387 distance > 20 388 ORDER BY 389 distance DESC 390 Limit 10;""" 391 query = f"SELECT {', '.join(columns)} FROM {table}" 392 if joins: 393 query += f" {' '.join(joins)}" 394 if where: 395 query += f" WHERE {where}" 396 if group_by: 397 query += f" GROUP BY {group_by}" 398 if having: 399 query += f" HAVING {having}" 400 if order_by: 401 query += f" ORDER BY {order_by}" 402 if limit: 403 query += f" LIMIT {limit}" 404 query += ";" 405 rows = self.query(query) 406 return rows 407 408 @staticmethod 409 def to_grid(data: list[dict], shrink_to_terminal: bool = True) -> str: 410 """Returns a tabular grid from `data`. 411 412 If `shrink_to_terminal` is `True`, the column widths of the grid will be reduced to fit within the current terminal. 413 """ 414 return griddy(data, "keys", shrink_to_terminal) 415 416 def update( 417 self, table: str, column: str, value: Any, where: str | None = None 418 ) -> int: 419 """Update `column` of `table` to `value` for rows satisfying the conditions in `where`. 420 421 If `where` is `None` all rows will be updated. 422 423 Returns the number of updated rows. 424 425 e.g. 426 >>> db = Databased() 427 >>> db.update("rides", "elevation", 100, "elevation < 100")""" 428 try: 429 if where: 430 self.query(f"UPDATE {table} SET {column} = ? WHERE {where};", (value,)) 431 else: 432 self.query(f"UPDATE {table} SET {column} = ?;", (value,)) 433 row_count = self.cursor.rowcount 434 self.logger.info( 435 f"Updated {row_count} rows in '{table}' table to '{column}' = '{value}' where '{where}'." 436 ) 437 return row_count 438 except Exception as e: 439 self.logger.exception( 440 f"Failed to update rows in '{table}' table to '{column}' = '{value}' where '{where}'." 441 ) 442 raise e 443 444 def vacuum(self) -> int: 445 """Reduce disk size of database after row/table deletion. 446 447 Returns space freed up in bytes.""" 448 size = self.path.size 449 self.query("VACUUM;") 450 return size - self.path.size
15class Databased: 16 """SQLite3 wrapper.""" 17 18 def __init__( 19 self, 20 dbpath: Pathish = "db.sqlite3", 21 connection_timeout: float = 10, 22 detect_types: bool = True, 23 enforce_foreign_keys: bool = True, 24 commit_on_close: bool = True, 25 logger_encoding: str = "utf-8", 26 logger_message_format: str = "{levelname}|-|{asctime}|-|{message}", 27 ): 28 """ """ 29 self.path = dbpath 30 self.connection_timeout = connection_timeout 31 self.connection = None 32 self._logger_init(logger_message_format, logger_encoding) 33 self.detect_types = detect_types 34 self.commit_on_close = commit_on_close 35 self.enforce_foreign_keys = enforce_foreign_keys 36 37 def __enter__(self): 38 self.connect() 39 return self 40 41 def __exit__(self, *args, **kwargs): 42 self.close() 43 44 @property 45 def commit_on_close(self) -> bool: 46 """Should commit database before closing connection when `self.close()` is called.""" 47 return self._commit_on_close 48 49 @commit_on_close.setter 50 def commit_on_close(self, should_commit_on_close: bool): 51 self._commit_on_close = should_commit_on_close 52 53 @property 54 def connected(self) -> bool: 55 """Whether this `Databased` instance is connected to the database file or not.""" 56 return self.connection is not None 57 58 @property 59 def connection_timeout(self) -> float: 60 """Changes to this property won't take effect until the current connection, if open, is closed and a new connection opened.""" 61 return self._connection_timeout 62 63 @connection_timeout.setter 64 def connection_timeout(self, timeout: float): 65 self._connection_timeout = timeout 66 67 @property 68 def detect_types(self) -> bool: 69 """Should use `detect_types=sqlite3.PARSE_DECLTYPES | sqlite3.PARSE_COLNAMES` when establishing a database connection. 70 71 Changes to this property won't take effect until the current connection, if open, is closed and a new connection opened. 72 """ 73 return self._detect_types 74 75 @detect_types.setter 76 def detect_types(self, should_detect: bool): 77 self._detect_types = should_detect 78 79 @property 80 def enforce_foreign_keys(self) -> bool: 81 return self._enforce_foreign_keys 82 83 @enforce_foreign_keys.setter 84 def enforce_foreign_keys(self, should_enforce: bool): 85 self._enforce_foreign_keys = should_enforce 86 self._set_foreign_key_enforcement() 87 88 @property 89 def indicies(self) -> list[str]: 90 """List of indicies for this database.""" 91 return [ 92 table["name"] 93 for table in self.query( 94 "SELECT name FROM sqlite_Schema WHERE type = 'index';" 95 ) 96 ] 97 98 @property 99 def name(self) -> str: 100 """The name of this database.""" 101 return self.path.stem 102 103 @property 104 def path(self) -> Pathier: 105 """The path to this database file.""" 106 return self._path 107 108 @path.setter 109 def path(self, new_path: Pathish): 110 """If `new_path` doesn't exist, it will be created (including parent folders).""" 111 self._path = Pathier(new_path) 112 if not self.path.exists(): 113 self.path.touch() 114 115 @property 116 def tables(self) -> list[str]: 117 """List of table names for this database.""" 118 return [ 119 table["name"] 120 for table in self.query( 121 "SELECT name FROM sqlite_Schema WHERE type = 'table' AND name NOT LIKE 'sqlite_%';" 122 ) 123 ] 124 125 @property 126 def views(self) -> list[str]: 127 """List of view for this database.""" 128 return [ 129 table["name"] 130 for table in self.query( 131 "SELECT name FROM sqlite_Schema WHERE type = 'view' AND name NOT LIKE 'sqlite_%';" 132 ) 133 ] 134 135 def _logger_init(self, message_format: str, encoding: str): 136 """:param: `message_format`: `{` style format string.""" 137 self.logger = logging.getLogger(self.name) 138 if not self.logger.hasHandlers(): 139 handler = logging.FileHandler( 140 str(self.path).replace(".", "") + ".log", encoding=encoding 141 ) 142 handler.setFormatter( 143 logging.Formatter( 144 message_format, style="{", datefmt="%m/%d/%Y %I:%M:%S %p" 145 ) 146 ) 147 self.logger.addHandler(handler) 148 self.logger.setLevel(logging.INFO) 149 150 def _set_foreign_key_enforcement(self): 151 if self.connection: 152 self.connection.execute( 153 f"pragma foreign_keys = {int(self.enforce_foreign_keys)};" 154 ) 155 156 def add_column(self, table: str, column_def: str): 157 """Add a column to `table`. 158 159 `column_def` should be in the form `{column_name} {type_name} {constraint}`. 160 161 i.e. 162 >>> db = Databased() 163 >>> db.add_column("rides", "num_stops INTEGER NOT NULL DEFAULT 0")""" 164 self.query(f"ALTER TABLE {table} ADD {column_def};") 165 166 def close(self): 167 """Disconnect from the database. 168 169 Does not call `commit()` for you unless the `commit_on_close` property is set to `True`. 170 """ 171 if self.connection: 172 if self.commit_on_close: 173 self.commit() 174 self.connection.close() 175 self.connection = None 176 177 def commit(self): 178 """Commit state of database.""" 179 if self.connection: 180 self.connection.commit() 181 self.logger.info("Committed successfully.") 182 else: 183 raise RuntimeError( 184 "Databased.commit(): Can't commit db with no open connection." 185 ) 186 187 def connect(self): 188 """Connect to the database.""" 189 self.connection = sqlite3.connect( 190 self.path, 191 detect_types=sqlite3.PARSE_DECLTYPES | sqlite3.PARSE_COLNAMES 192 if self.detect_types 193 else 0, 194 timeout=self.connection_timeout, 195 ) 196 self._set_foreign_key_enforcement() 197 self.connection.row_factory = dict_factory 198 199 def count( 200 self, 201 table: str, 202 column: str = "*", 203 where: str | None = None, 204 distinct: bool = False, 205 ) -> int: 206 """Return number of matching rows in `table` table. 207 208 Equivalent to: 209 >>> SELECT COUNT({distinct} {column}) FROM {table} {where};""" 210 query = ( 211 f"SELECT COUNT( {('DISTINCT' if distinct else '')} {column}) FROM {table}" 212 ) 213 if where: 214 query += f" WHERE {where}" 215 query += ";" 216 return int(list(self.query(query)[0].values())[0]) 217 218 def create_table(self, table: str, *column_defs: str): 219 """Create a table if it doesn't exist. 220 221 #### :params: 222 223 `table`: Name of the table to create. 224 225 `column_defs`: Any number of column names and their definitions in proper Sqlite3 sytax. 226 i.e. `"column_name TEXT UNIQUE"` or `"column_name INTEGER PRIMARY KEY"` etc.""" 227 columns = ", ".join(column_defs) 228 result = self.query(f"CREATE TABLE IF NOT EXISTS {table} ({columns});") 229 self.logger.info(f"'{table}' table created.") 230 231 def delete(self, table: str, where: str | None = None) -> int: 232 """Delete rows from `table` that satisfy the given `where` clause. 233 234 If `where` is `None`, all rows will be deleted. 235 236 Returns the number of deleted rows. 237 238 e.g. 239 >>> db = Databased() 240 >>> db.delete("rides", "distance < 5 AND average_speed < 7")""" 241 try: 242 if where: 243 self.query(f"DELETE FROM {table} WHERE {where};") 244 else: 245 self.query(f"DELETE FROM {table};") 246 row_count = self.cursor.rowcount 247 self.logger.info( 248 f"Deleted {row_count} rows from '{table}' where '{where}'." 249 ) 250 return row_count 251 except Exception as e: 252 self.logger.exception( 253 f"Error deleting rows from '{table}' where '{where}'." 254 ) 255 raise e 256 257 def describe(self, table: str) -> list[dict]: 258 """Returns information about `table`.""" 259 return self.query(f"pragma table_info('{table}');") 260 261 def drop_column(self, table: str, column: str): 262 """Drop `column` from `table`.""" 263 self.query(f"ALTER TABLE {table} DROP {column};") 264 265 def drop_table(self, table: str) -> bool: 266 """Drop `table` from the database. 267 268 Returns `True` if successful, `False` if not.""" 269 try: 270 self.query(f"DROP TABLE {table};") 271 self.logger.info(f"Dropped table '{table}'.") 272 return True 273 except Exception as e: 274 print(f"{type(e).__name__}: {e}") 275 self.logger.error(f"Failed to drop table '{table}'.") 276 return False 277 278 def execute_script(self, path: Pathish, encoding: str = "utf-8") -> list[dict]: 279 """Execute sql script located at `path`.""" 280 if not self.connected: 281 self.connect() 282 assert self.connection 283 script = Pathier(path).read_text(encoding).replace("\n", " ") 284 return self.query(script) 285 286 def get_columns(self, table: str) -> tuple[str, ...]: 287 """Returns a list of column names in `table`.""" 288 return tuple( 289 (column["name"] for column in self.query(f"pragma table_info('{table}');")) 290 ) 291 292 def insert( 293 self, table: str, columns: tuple[str, ...], values: list[tuple[Any, ...]] 294 ) -> int: 295 """Insert rows of `values` into `columns` of `table`. 296 297 Each `tuple` in `values` corresponds to an individual row that is to be inserted. 298 """ 299 max_row_count = 900 300 column_list = "(" + ", ".join(columns) + ")" 301 row_count = 0 302 for i in range(0, len(values), max_row_count): 303 chunk = values[i : i + max_row_count] 304 placeholder = ( 305 "(" + "),(".join((", ".join(("?" for _ in row)) for row in chunk)) + ")" 306 ) 307 logger_values = "\n".join( 308 ( 309 "'(" + ", ".join((str(value) for value in row)) + ")'" 310 for row in chunk 311 ) 312 ) 313 flattened_values = tuple((value for row in chunk for value in row)) 314 try: 315 self.query( 316 f"INSERT INTO {table} {column_list} VALUES {placeholder};", 317 flattened_values, 318 ) 319 self.logger.info( 320 f"Inserted into '{column_list}' columns of '{table}' table values \n{logger_values}." 321 ) 322 row_count += self.cursor.rowcount 323 except Exception as e: 324 self.logger.exception( 325 f"Error inserting into '{column_list}' columns of '{table}' table values \n{logger_values}." 326 ) 327 raise e 328 return row_count 329 330 def query(self, query_: str, parameters: tuple[Any, ...] = tuple()) -> list[dict]: 331 """Execute an SQL query and return the results. 332 333 Ensures that the database connection is opened before executing the command. 334 335 The cursor used to execute the query will be available through `self.cursor` until the next time `self.query()` is called. 336 """ 337 if not self.connected: 338 self.connect() 339 assert self.connection 340 self.cursor = self.connection.cursor() 341 self.cursor.execute(query_, parameters) 342 return self.cursor.fetchall() 343 344 def rename_column(self, table: str, column_to_rename: str, new_column_name: str): 345 """Rename a column in `table`.""" 346 self.query( 347 f"ALTER TABLE {table} RENAME {column_to_rename} TO {new_column_name};" 348 ) 349 350 def rename_table(self, table_to_rename: str, new_table_name: str): 351 """Rename a table.""" 352 self.query(f"ALTER TABLE {table_to_rename} RENAME TO {new_table_name};") 353 354 def select( 355 self, 356 table: str, 357 columns: list[str] = ["*"], 358 joins: list[str] | None = None, 359 where: str | None = None, 360 group_by: str | None = None, 361 having: str | None = None, 362 order_by: str | None = None, 363 limit: int | str | None = None, 364 ) -> list[dict]: 365 """Return rows for given criteria. 366 367 For complex queries, use the `databased.query()` method. 368 369 Parameters `where`, `group_by`, `having`, `order_by`, and `limit` should not have 370 their corresponding key word in their string, but should otherwise be valid SQL. 371 372 `joins` should contain their key word (`INNER JOIN`, `LEFT JOIN`) in addition to the rest of the sub-statement. 373 374 >>> Databased().select( 375 "bike_rides", 376 "id, date, distance, moving_time, AVG(distance/moving_time) as average_speed", 377 where="distance > 20", 378 order_by="distance", 379 desc=True, 380 limit=10 381 ) 382 executes the query: 383 >>> SELECT 384 id, date, distance, moving_time, AVG(distance/moving_time) as average_speed 385 FROM 386 bike_rides 387 WHERE 388 distance > 20 389 ORDER BY 390 distance DESC 391 Limit 10;""" 392 query = f"SELECT {', '.join(columns)} FROM {table}" 393 if joins: 394 query += f" {' '.join(joins)}" 395 if where: 396 query += f" WHERE {where}" 397 if group_by: 398 query += f" GROUP BY {group_by}" 399 if having: 400 query += f" HAVING {having}" 401 if order_by: 402 query += f" ORDER BY {order_by}" 403 if limit: 404 query += f" LIMIT {limit}" 405 query += ";" 406 rows = self.query(query) 407 return rows 408 409 @staticmethod 410 def to_grid(data: list[dict], shrink_to_terminal: bool = True) -> str: 411 """Returns a tabular grid from `data`. 412 413 If `shrink_to_terminal` is `True`, the column widths of the grid will be reduced to fit within the current terminal. 414 """ 415 return griddy(data, "keys", shrink_to_terminal) 416 417 def update( 418 self, table: str, column: str, value: Any, where: str | None = None 419 ) -> int: 420 """Update `column` of `table` to `value` for rows satisfying the conditions in `where`. 421 422 If `where` is `None` all rows will be updated. 423 424 Returns the number of updated rows. 425 426 e.g. 427 >>> db = Databased() 428 >>> db.update("rides", "elevation", 100, "elevation < 100")""" 429 try: 430 if where: 431 self.query(f"UPDATE {table} SET {column} = ? WHERE {where};", (value,)) 432 else: 433 self.query(f"UPDATE {table} SET {column} = ?;", (value,)) 434 row_count = self.cursor.rowcount 435 self.logger.info( 436 f"Updated {row_count} rows in '{table}' table to '{column}' = '{value}' where '{where}'." 437 ) 438 return row_count 439 except Exception as e: 440 self.logger.exception( 441 f"Failed to update rows in '{table}' table to '{column}' = '{value}' where '{where}'." 442 ) 443 raise e 444 445 def vacuum(self) -> int: 446 """Reduce disk size of database after row/table deletion. 447 448 Returns space freed up in bytes.""" 449 size = self.path.size 450 self.query("VACUUM;") 451 return size - self.path.size
SQLite3 wrapper.
18 def __init__( 19 self, 20 dbpath: Pathish = "db.sqlite3", 21 connection_timeout: float = 10, 22 detect_types: bool = True, 23 enforce_foreign_keys: bool = True, 24 commit_on_close: bool = True, 25 logger_encoding: str = "utf-8", 26 logger_message_format: str = "{levelname}|-|{asctime}|-|{message}", 27 ): 28 """ """ 29 self.path = dbpath 30 self.connection_timeout = connection_timeout 31 self.connection = None 32 self._logger_init(logger_message_format, logger_encoding) 33 self.detect_types = detect_types 34 self.commit_on_close = commit_on_close 35 self.enforce_foreign_keys = enforce_foreign_keys
If new_path
doesn't exist, it will be created (including parent folders).
Changes to this property won't take effect until the current connection, if open, is closed and a new connection opened.
Should use detect_types=sqlite3.PARSE_DECLTYPES | sqlite3.PARSE_COLNAMES
when establishing a database connection.
Changes to this property won't take effect until the current connection, if open, is closed and a new connection opened.
156 def add_column(self, table: str, column_def: str): 157 """Add a column to `table`. 158 159 `column_def` should be in the form `{column_name} {type_name} {constraint}`. 160 161 i.e. 162 >>> db = Databased() 163 >>> db.add_column("rides", "num_stops INTEGER NOT NULL DEFAULT 0")""" 164 self.query(f"ALTER TABLE {table} ADD {column_def};")
Add a column to table
.
column_def
should be in the form {column_name} {type_name} {constraint}
.
i.e.
>>> db = Databased()
>>> db.add_column("rides", "num_stops INTEGER NOT NULL DEFAULT 0")
166 def close(self): 167 """Disconnect from the database. 168 169 Does not call `commit()` for you unless the `commit_on_close` property is set to `True`. 170 """ 171 if self.connection: 172 if self.commit_on_close: 173 self.commit() 174 self.connection.close() 175 self.connection = None
Disconnect from the database.
Does not call commit()
for you unless the commit_on_close
property is set to True
.
177 def commit(self): 178 """Commit state of database.""" 179 if self.connection: 180 self.connection.commit() 181 self.logger.info("Committed successfully.") 182 else: 183 raise RuntimeError( 184 "Databased.commit(): Can't commit db with no open connection." 185 )
Commit state of database.
187 def connect(self): 188 """Connect to the database.""" 189 self.connection = sqlite3.connect( 190 self.path, 191 detect_types=sqlite3.PARSE_DECLTYPES | sqlite3.PARSE_COLNAMES 192 if self.detect_types 193 else 0, 194 timeout=self.connection_timeout, 195 ) 196 self._set_foreign_key_enforcement() 197 self.connection.row_factory = dict_factory
Connect to the database.
199 def count( 200 self, 201 table: str, 202 column: str = "*", 203 where: str | None = None, 204 distinct: bool = False, 205 ) -> int: 206 """Return number of matching rows in `table` table. 207 208 Equivalent to: 209 >>> SELECT COUNT({distinct} {column}) FROM {table} {where};""" 210 query = ( 211 f"SELECT COUNT( {('DISTINCT' if distinct else '')} {column}) FROM {table}" 212 ) 213 if where: 214 query += f" WHERE {where}" 215 query += ";" 216 return int(list(self.query(query)[0].values())[0])
Return number of matching rows in table
table.
Equivalent to:
>>> SELECT COUNT({distinct} {column}) FROM {table} {where};
218 def create_table(self, table: str, *column_defs: str): 219 """Create a table if it doesn't exist. 220 221 #### :params: 222 223 `table`: Name of the table to create. 224 225 `column_defs`: Any number of column names and their definitions in proper Sqlite3 sytax. 226 i.e. `"column_name TEXT UNIQUE"` or `"column_name INTEGER PRIMARY KEY"` etc.""" 227 columns = ", ".join(column_defs) 228 result = self.query(f"CREATE TABLE IF NOT EXISTS {table} ({columns});") 229 self.logger.info(f"'{table}' table created.")
Create a table if it doesn't exist.
:params:
table
: Name of the table to create.
column_defs
: Any number of column names and their definitions in proper Sqlite3 sytax.
i.e. "column_name TEXT UNIQUE"
or "column_name INTEGER PRIMARY KEY"
etc.
231 def delete(self, table: str, where: str | None = None) -> int: 232 """Delete rows from `table` that satisfy the given `where` clause. 233 234 If `where` is `None`, all rows will be deleted. 235 236 Returns the number of deleted rows. 237 238 e.g. 239 >>> db = Databased() 240 >>> db.delete("rides", "distance < 5 AND average_speed < 7")""" 241 try: 242 if where: 243 self.query(f"DELETE FROM {table} WHERE {where};") 244 else: 245 self.query(f"DELETE FROM {table};") 246 row_count = self.cursor.rowcount 247 self.logger.info( 248 f"Deleted {row_count} rows from '{table}' where '{where}'." 249 ) 250 return row_count 251 except Exception as e: 252 self.logger.exception( 253 f"Error deleting rows from '{table}' where '{where}'." 254 ) 255 raise e
Delete rows from table
that satisfy the given where
clause.
If where
is None
, all rows will be deleted.
Returns the number of deleted rows.
e.g.
>>> db = Databased()
>>> db.delete("rides", "distance < 5 AND average_speed < 7")
257 def describe(self, table: str) -> list[dict]: 258 """Returns information about `table`.""" 259 return self.query(f"pragma table_info('{table}');")
Returns information about table
.
261 def drop_column(self, table: str, column: str): 262 """Drop `column` from `table`.""" 263 self.query(f"ALTER TABLE {table} DROP {column};")
Drop column
from table
.
265 def drop_table(self, table: str) -> bool: 266 """Drop `table` from the database. 267 268 Returns `True` if successful, `False` if not.""" 269 try: 270 self.query(f"DROP TABLE {table};") 271 self.logger.info(f"Dropped table '{table}'.") 272 return True 273 except Exception as e: 274 print(f"{type(e).__name__}: {e}") 275 self.logger.error(f"Failed to drop table '{table}'.") 276 return False
Drop table
from the database.
Returns True
if successful, False
if not.
278 def execute_script(self, path: Pathish, encoding: str = "utf-8") -> list[dict]: 279 """Execute sql script located at `path`.""" 280 if not self.connected: 281 self.connect() 282 assert self.connection 283 script = Pathier(path).read_text(encoding).replace("\n", " ") 284 return self.query(script)
Execute sql script located at path
.
286 def get_columns(self, table: str) -> tuple[str, ...]: 287 """Returns a list of column names in `table`.""" 288 return tuple( 289 (column["name"] for column in self.query(f"pragma table_info('{table}');")) 290 )
Returns a list of column names in table
.
292 def insert( 293 self, table: str, columns: tuple[str, ...], values: list[tuple[Any, ...]] 294 ) -> int: 295 """Insert rows of `values` into `columns` of `table`. 296 297 Each `tuple` in `values` corresponds to an individual row that is to be inserted. 298 """ 299 max_row_count = 900 300 column_list = "(" + ", ".join(columns) + ")" 301 row_count = 0 302 for i in range(0, len(values), max_row_count): 303 chunk = values[i : i + max_row_count] 304 placeholder = ( 305 "(" + "),(".join((", ".join(("?" for _ in row)) for row in chunk)) + ")" 306 ) 307 logger_values = "\n".join( 308 ( 309 "'(" + ", ".join((str(value) for value in row)) + ")'" 310 for row in chunk 311 ) 312 ) 313 flattened_values = tuple((value for row in chunk for value in row)) 314 try: 315 self.query( 316 f"INSERT INTO {table} {column_list} VALUES {placeholder};", 317 flattened_values, 318 ) 319 self.logger.info( 320 f"Inserted into '{column_list}' columns of '{table}' table values \n{logger_values}." 321 ) 322 row_count += self.cursor.rowcount 323 except Exception as e: 324 self.logger.exception( 325 f"Error inserting into '{column_list}' columns of '{table}' table values \n{logger_values}." 326 ) 327 raise e 328 return row_count
Insert rows of values
into columns
of table
.
Each tuple
in values
corresponds to an individual row that is to be inserted.
330 def query(self, query_: str, parameters: tuple[Any, ...] = tuple()) -> list[dict]: 331 """Execute an SQL query and return the results. 332 333 Ensures that the database connection is opened before executing the command. 334 335 The cursor used to execute the query will be available through `self.cursor` until the next time `self.query()` is called. 336 """ 337 if not self.connected: 338 self.connect() 339 assert self.connection 340 self.cursor = self.connection.cursor() 341 self.cursor.execute(query_, parameters) 342 return self.cursor.fetchall()
Execute an SQL query and return the results.
Ensures that the database connection is opened before executing the command.
The cursor used to execute the query will be available through self.cursor
until the next time self.query()
is called.
344 def rename_column(self, table: str, column_to_rename: str, new_column_name: str): 345 """Rename a column in `table`.""" 346 self.query( 347 f"ALTER TABLE {table} RENAME {column_to_rename} TO {new_column_name};" 348 )
Rename a column in table
.
350 def rename_table(self, table_to_rename: str, new_table_name: str): 351 """Rename a table.""" 352 self.query(f"ALTER TABLE {table_to_rename} RENAME TO {new_table_name};")
Rename a table.
354 def select( 355 self, 356 table: str, 357 columns: list[str] = ["*"], 358 joins: list[str] | None = None, 359 where: str | None = None, 360 group_by: str | None = None, 361 having: str | None = None, 362 order_by: str | None = None, 363 limit: int | str | None = None, 364 ) -> list[dict]: 365 """Return rows for given criteria. 366 367 For complex queries, use the `databased.query()` method. 368 369 Parameters `where`, `group_by`, `having`, `order_by`, and `limit` should not have 370 their corresponding key word in their string, but should otherwise be valid SQL. 371 372 `joins` should contain their key word (`INNER JOIN`, `LEFT JOIN`) in addition to the rest of the sub-statement. 373 374 >>> Databased().select( 375 "bike_rides", 376 "id, date, distance, moving_time, AVG(distance/moving_time) as average_speed", 377 where="distance > 20", 378 order_by="distance", 379 desc=True, 380 limit=10 381 ) 382 executes the query: 383 >>> SELECT 384 id, date, distance, moving_time, AVG(distance/moving_time) as average_speed 385 FROM 386 bike_rides 387 WHERE 388 distance > 20 389 ORDER BY 390 distance DESC 391 Limit 10;""" 392 query = f"SELECT {', '.join(columns)} FROM {table}" 393 if joins: 394 query += f" {' '.join(joins)}" 395 if where: 396 query += f" WHERE {where}" 397 if group_by: 398 query += f" GROUP BY {group_by}" 399 if having: 400 query += f" HAVING {having}" 401 if order_by: 402 query += f" ORDER BY {order_by}" 403 if limit: 404 query += f" LIMIT {limit}" 405 query += ";" 406 rows = self.query(query) 407 return rows
Return rows for given criteria.
For complex queries, use the databased.query()
method.
Parameters where
, group_by
, having
, order_by
, and limit
should not have
their corresponding key word in their string, but should otherwise be valid SQL.
joins
should contain their key word (INNER JOIN
, LEFT JOIN
) in addition to the rest of the sub-statement.
>>> Databased().select(
"bike_rides",
"id, date, distance, moving_time, AVG(distance/moving_time) as average_speed",
where="distance > 20",
order_by="distance",
desc=True,
limit=10
)
executes the query:
>>> SELECT
id, date, distance, moving_time, AVG(distance/moving_time) as average_speed
FROM
bike_rides
WHERE
distance > 20
ORDER BY
distance DESC
Limit 10;
409 @staticmethod 410 def to_grid(data: list[dict], shrink_to_terminal: bool = True) -> str: 411 """Returns a tabular grid from `data`. 412 413 If `shrink_to_terminal` is `True`, the column widths of the grid will be reduced to fit within the current terminal. 414 """ 415 return griddy(data, "keys", shrink_to_terminal)
Returns a tabular grid from data
.
If shrink_to_terminal
is True
, the column widths of the grid will be reduced to fit within the current terminal.
417 def update( 418 self, table: str, column: str, value: Any, where: str | None = None 419 ) -> int: 420 """Update `column` of `table` to `value` for rows satisfying the conditions in `where`. 421 422 If `where` is `None` all rows will be updated. 423 424 Returns the number of updated rows. 425 426 e.g. 427 >>> db = Databased() 428 >>> db.update("rides", "elevation", 100, "elevation < 100")""" 429 try: 430 if where: 431 self.query(f"UPDATE {table} SET {column} = ? WHERE {where};", (value,)) 432 else: 433 self.query(f"UPDATE {table} SET {column} = ?;", (value,)) 434 row_count = self.cursor.rowcount 435 self.logger.info( 436 f"Updated {row_count} rows in '{table}' table to '{column}' = '{value}' where '{where}'." 437 ) 438 return row_count 439 except Exception as e: 440 self.logger.exception( 441 f"Failed to update rows in '{table}' table to '{column}' = '{value}' where '{where}'." 442 ) 443 raise e
Update column
of table
to value
for rows satisfying the conditions in where
.
If where
is None
all rows will be updated.
Returns the number of updated rows.
e.g.
>>> db = Databased()
>>> db.update("rides", "elevation", 100, "elevation < 100")
445 def vacuum(self) -> int: 446 """Reduce disk size of database after row/table deletion. 447 448 Returns space freed up in bytes.""" 449 size = self.path.size 450 self.query("VACUUM;") 451 return size - self.path.size
Reduce disk size of database after row/table deletion.
Returns space freed up in bytes.