databased.databased
1import logging 2import sqlite3 3from typing import Any 4 5from griddle import griddy 6from pathier import Pathier, Pathish 7 8 9def dict_factory(cursor: sqlite3.Cursor, row: tuple) -> dict: 10 fields = [column[0] for column in cursor.description] 11 return {column: value for column, value in zip(fields, row)} 12 13 14class Databased: 15 """SQLite3 wrapper.""" 16 17 def __init__( 18 self, 19 dbpath: Pathish = "db.sqlite3", 20 connection_timeout: float = 10, 21 detect_types: bool = True, 22 enforce_foreign_keys: bool = True, 23 commit_on_close: bool = True, 24 logger_encoding: str = "utf-8", 25 logger_message_format: str = "{levelname}|-|{asctime}|-|{message}", 26 ): 27 """ """ 28 self.path = dbpath 29 self.connection_timeout = connection_timeout 30 self.connection = None 31 self._logger_init(logger_message_format, logger_encoding) 32 self.detect_types = detect_types 33 self.commit_on_close = commit_on_close 34 self.enforce_foreign_keys = enforce_foreign_keys 35 36 def __enter__(self): 37 self.connect() 38 return self 39 40 def __exit__(self, *args, **kwargs): 41 self.close() 42 43 @property 44 def commit_on_close(self) -> bool: 45 """Should commit database before closing connection when `self.close()` is called.""" 46 return self._commit_on_close 47 48 @commit_on_close.setter 49 def commit_on_close(self, should_commit_on_close: bool): 50 self._commit_on_close = should_commit_on_close 51 52 @property 53 def connected(self) -> bool: 54 """Whether this `Databased` instance is connected to the database file or not.""" 55 return self.connection is not None 56 57 @property 58 def connection_timeout(self) -> float: 59 """Changes to this property won't take effect until the current connection, if open, is closed and a new connection opened.""" 60 return self._connection_timeout 61 62 @connection_timeout.setter 63 def connection_timeout(self, timeout: float): 64 self._connection_timeout = timeout 65 66 @property 67 def detect_types(self) -> bool: 68 """Should use `detect_types=sqlite3.PARSE_DECLTYPES | sqlite3.PARSE_COLNAMES` when establishing a database connection. 69 70 Changes to this property won't take effect until the current connection, if open, is closed and a new connection opened. 71 """ 72 return self._detect_types 73 74 @detect_types.setter 75 def detect_types(self, should_detect: bool): 76 self._detect_types = should_detect 77 78 @property 79 def enforce_foreign_keys(self) -> bool: 80 return self._enforce_foreign_keys 81 82 @enforce_foreign_keys.setter 83 def enforce_foreign_keys(self, should_enforce: bool): 84 self._enforce_foreign_keys = should_enforce 85 self._set_foreign_key_enforcement() 86 87 @property 88 def name(self) -> str: 89 """The name of this database.""" 90 return self.path.stem 91 92 @property 93 def path(self) -> Pathier: 94 """The path to this database file.""" 95 return self._path 96 97 @path.setter 98 def path(self, new_path: Pathish): 99 """If `new_path` doesn't exist, it will be created (including parent folders).""" 100 self._path = Pathier(new_path) 101 if not self.path.exists(): 102 self.path.touch() 103 104 @property 105 def tables(self) -> list[str]: 106 """List of table names for this database.""" 107 return [ 108 table["name"] 109 for table in self.query( 110 "SELECT name FROM sqlite_Schema WHERE type = 'table' AND name NOT LIKE 'sqlite_%';" 111 ) 112 ] 113 114 def _logger_init(self, message_format: str, encoding: str): 115 """:param: `message_format`: `{` style format string.""" 116 self.logger = logging.getLogger(self.name) 117 if not self.logger.hasHandlers(): 118 handler = logging.FileHandler( 119 str(self.path).replace(".", "") + ".log", encoding=encoding 120 ) 121 handler.setFormatter( 122 logging.Formatter( 123 message_format, style="{", datefmt="%m/%d/%Y %I:%M:%S %p" 124 ) 125 ) 126 self.logger.addHandler(handler) 127 self.logger.setLevel(logging.INFO) 128 129 def _set_foreign_key_enforcement(self): 130 if self.connection: 131 self.connection.execute( 132 f"pragma foreign_keys = {int(self.enforce_foreign_keys)};" 133 ) 134 135 def add_column(self, table: str, column_def: str): 136 """Add a column to `table`. 137 138 `column_def` should be in the form `{column_name} {type_name} {constraint}`. 139 140 i.e. 141 >>> db = Databased() 142 >>> db.add_column("rides", "num_stops INTEGER NOT NULL DEFAULT 0")""" 143 self.query(f"ALTER TABLE {table} ADD {column_def};") 144 145 def close(self): 146 """Disconnect from the database. 147 148 Does not call `commit()` for you unless the `commit_on_close` property is set to `True`. 149 """ 150 if self.connection: 151 if self.commit_on_close: 152 self.commit() 153 self.connection.close() 154 self.connection = None 155 156 def commit(self): 157 """Commit state of database.""" 158 if self.connection: 159 self.connection.commit() 160 self.logger.info("Committed successfully.") 161 else: 162 raise RuntimeError( 163 "Databased.commit(): Can't commit db with no open connection." 164 ) 165 166 def connect(self): 167 """Connect to the database.""" 168 self.connection = sqlite3.connect( 169 self.path, 170 detect_types=sqlite3.PARSE_DECLTYPES | sqlite3.PARSE_COLNAMES 171 if self.detect_types 172 else 0, 173 timeout=self.connection_timeout, 174 ) 175 self._set_foreign_key_enforcement() 176 self.connection.row_factory = dict_factory 177 178 def count( 179 self, 180 table: str, 181 column: str = "*", 182 where: str | None = None, 183 distinct: bool = False, 184 ) -> int: 185 """Return number of matching rows in `table` table. 186 187 Equivalent to: 188 >>> SELECT COUNT({distinct} {column}) FROM {table} {where};""" 189 query = ( 190 f"SELECT COUNT( {('DISTINCT' if distinct else '')} {column}) FROM {table}" 191 ) 192 if where: 193 query += f" WHERE {where}" 194 query += ";" 195 return int(list(self.query(query)[0].values())[0]) 196 197 def create_table(self, table: str, *column_defs: str): 198 """Create a table if it doesn't exist. 199 200 #### :params: 201 202 `table`: Name of the table to create. 203 204 `column_defs`: Any number of column names and their definitions in proper Sqlite3 sytax. 205 i.e. `"column_name TEXT UNIQUE"` or `"column_name INTEGER PRIMARY KEY"` etc.""" 206 columns = ", ".join(column_defs) 207 result = self.query(f"CREATE TABLE IF NOT EXISTS {table} ({columns});") 208 self.logger.info(f"'{table}' table created.") 209 210 def delete(self, table: str, where: str | None = None) -> int: 211 """Delete rows from `table` that satisfy the given `where` clause. 212 213 If `where` is `None`, all rows will be deleted. 214 215 Returns the number of deleted rows. 216 217 e.g. 218 >>> db = Databased() 219 >>> db.delete("rides", "distance < 5 AND average_speed < 7")""" 220 try: 221 if where: 222 self.query(f"DELETE FROM {table} WHERE {where};") 223 else: 224 self.query(f"DELETE FROM {table};") 225 row_count = self.cursor.rowcount 226 self.logger.info( 227 f"Deleted {row_count} rows from '{table}' where '{where}'." 228 ) 229 return row_count 230 except Exception as e: 231 self.logger.exception( 232 f"Error deleting rows from '{table}' where '{where}'." 233 ) 234 raise e 235 236 def describe(self, table: str) -> list[dict]: 237 """Returns information about `table`.""" 238 return self.query(f"pragma table_info('{table}');") 239 240 def drop_column(self, table: str, column: str): 241 """Drop `column` from `table`.""" 242 self.query(f"ALTER TABLE {table} DROP {column};") 243 244 def drop_table(self, table: str) -> bool: 245 """Drop `table` from the database. 246 247 Returns `True` if successful, `False` if not.""" 248 try: 249 self.query(f"DROP TABLE {table};") 250 self.logger.info(f"Dropped table '{table}'.") 251 return True 252 except Exception as e: 253 print(f"{type(e).__name__}: {e}") 254 self.logger.error(f"Failed to drop table '{table}'.") 255 return False 256 257 def execute_script(self, path: Pathish, encoding: str = "utf-8") -> list[dict]: 258 """Execute sql script located at `path`.""" 259 if not self.connected: 260 self.connect() 261 assert self.connection 262 script = Pathier(path).read_text(encoding).replace("\n", " ") 263 return self.query(script) 264 265 def get_columns(self, table: str) -> list[str]: 266 """Returns a list of column names in `table`.""" 267 return [ 268 column["name"] for column in self.query(f"pragma table_info('{table}');") 269 ] 270 271 def insert( 272 self, table: str, columns: tuple[str, ...], values: list[tuple[Any, ...]] 273 ) -> int: 274 """Insert rows of `values` into `columns` of `table`. 275 276 Each `tuple` in `values` corresponds to an individual row that is to be inserted. 277 """ 278 max_row_count = 900 279 column_list = "(" + ", ".join(columns) + ")" 280 row_count = 0 281 for i in range(0, len(values), max_row_count): 282 chunk = values[i : i + max_row_count] 283 placeholder = ( 284 "(" + "),(".join((", ".join(("?" for _ in row)) for row in chunk)) + ")" 285 ) 286 logger_values = "\n".join( 287 ( 288 "'(" + ", ".join((str(value) for value in row)) + ")'" 289 for row in chunk 290 ) 291 ) 292 flattened_values = tuple((value for row in chunk for value in row)) 293 try: 294 self.query( 295 f"INSERT INTO {table} {column_list} VALUES {placeholder};", 296 flattened_values, 297 ) 298 self.logger.info( 299 f"Inserted into '{column_list}' columns of '{table}' table values \n{logger_values}." 300 ) 301 row_count += self.cursor.rowcount 302 except Exception as e: 303 self.logger.exception( 304 f"Error inserting into '{column_list}' columns of '{table}' table values \n{logger_values}." 305 ) 306 raise e 307 return row_count 308 309 def query(self, query_: str, parameters: tuple[Any, ...] = tuple()) -> list[dict]: 310 """Execute an SQL query and return the results. 311 312 Ensures that the database connection is opened before executing the command. 313 314 The cursor used to execute the query will be available through `self.cursor` until the next time `self.query()` is called. 315 """ 316 if not self.connected: 317 self.connect() 318 assert self.connection 319 self.cursor = self.connection.cursor() 320 self.cursor.execute(query_, parameters) 321 return self.cursor.fetchall() 322 323 def rename_column(self, table: str, column_to_rename: str, new_column_name: str): 324 """Rename a column in `table`.""" 325 self.query( 326 f"ALTER TABLE {table} RENAME {column_to_rename} TO {new_column_name};" 327 ) 328 329 def rename_table(self, table_to_rename: str, new_table_name: str): 330 """Rename a table.""" 331 self.query(f"ALTER TABLE {table_to_rename} RENAME TO {new_table_name};") 332 333 def select( 334 self, 335 table: str, 336 columns: list[str] = ["*"], 337 joins: list[str] | None = None, 338 where: str | None = None, 339 group_by: str | None = None, 340 having: str | None = None, 341 order_by: str | None = None, 342 limit: int | str | None = None, 343 ) -> list[dict]: 344 """Return rows for given criteria. 345 346 For complex queries, use the `databased.query()` method. 347 348 Parameters `where`, `group_by`, `having`, `order_by`, and `limit` should not have 349 their corresponding key word in their string, but should otherwise be valid SQL. 350 351 `joins` should contain their key word (`INNER JOIN`, `LEFT JOIN`) in addition to the rest of the sub-statement. 352 353 >>> Databased().select( 354 "bike_rides", 355 "id, date, distance, moving_time, AVG(distance/moving_time) as average_speed", 356 where="distance > 20", 357 order_by="distance", 358 desc=True, 359 limit=10 360 ) 361 executes the query: 362 >>> SELECT 363 id, date, distance, moving_time, AVG(distance/moving_time) as average_speed 364 FROM 365 bike_rides 366 WHERE 367 distance > 20 368 ORDER BY 369 distance DESC 370 Limit 10;""" 371 query = f"SELECT {', '.join(columns)} FROM {table}" 372 if joins: 373 query += f" {' '.join(joins)}" 374 if where: 375 query += f" WHERE {where}" 376 if group_by: 377 query += f" GROUP BY {group_by}" 378 if having: 379 query += f" HAVING {having}" 380 if order_by: 381 query += f" ORDER BY {order_by}" 382 if limit: 383 query += f" LIMIT {limit}" 384 query += ";" 385 rows = self.query(query) 386 return rows 387 388 @staticmethod 389 def to_grid(data: list[dict], shrink_to_terminal: bool = True) -> str: 390 """Returns a tabular grid from `data`. 391 392 If `shrink_to_terminal` is `True`, the column widths of the grid will be reduced to fit within the current terminal. 393 """ 394 return griddy(data, "keys", shrink_to_terminal) 395 396 def update( 397 self, table: str, column: str, value: Any, where: str | None = None 398 ) -> int: 399 """Update `column` of `table` to `value` for rows satisfying the conditions in `where`. 400 401 If `where` is `None` all rows will be updated. 402 403 Returns the number of updated rows. 404 405 e.g. 406 >>> db = Databased() 407 >>> db.update("rides", "elevation", 100, "elevation < 100")""" 408 try: 409 if where: 410 self.query(f"UPDATE {table} SET {column} = ? WHERE {where};", (value,)) 411 else: 412 self.query(f"UPDATE {table} SET {column} = ?;", (value,)) 413 row_count = self.cursor.rowcount 414 self.logger.info( 415 f"Updated {row_count} rows in '{table}' table to '{column}' = '{value}' where '{where}'." 416 ) 417 return row_count 418 except Exception as e: 419 self.logger.exception( 420 f"Failed to update rows in '{table}' table to '{column}' = '{value}' where '{where}'." 421 ) 422 raise e 423 424 def vacuum(self) -> int: 425 """Reduce disk size of database after row/table deletion. 426 427 Returns space freed up in bytes.""" 428 size = self.path.size 429 self.query("VACUUM;") 430 return size - self.path.size
15class Databased: 16 """SQLite3 wrapper.""" 17 18 def __init__( 19 self, 20 dbpath: Pathish = "db.sqlite3", 21 connection_timeout: float = 10, 22 detect_types: bool = True, 23 enforce_foreign_keys: bool = True, 24 commit_on_close: bool = True, 25 logger_encoding: str = "utf-8", 26 logger_message_format: str = "{levelname}|-|{asctime}|-|{message}", 27 ): 28 """ """ 29 self.path = dbpath 30 self.connection_timeout = connection_timeout 31 self.connection = None 32 self._logger_init(logger_message_format, logger_encoding) 33 self.detect_types = detect_types 34 self.commit_on_close = commit_on_close 35 self.enforce_foreign_keys = enforce_foreign_keys 36 37 def __enter__(self): 38 self.connect() 39 return self 40 41 def __exit__(self, *args, **kwargs): 42 self.close() 43 44 @property 45 def commit_on_close(self) -> bool: 46 """Should commit database before closing connection when `self.close()` is called.""" 47 return self._commit_on_close 48 49 @commit_on_close.setter 50 def commit_on_close(self, should_commit_on_close: bool): 51 self._commit_on_close = should_commit_on_close 52 53 @property 54 def connected(self) -> bool: 55 """Whether this `Databased` instance is connected to the database file or not.""" 56 return self.connection is not None 57 58 @property 59 def connection_timeout(self) -> float: 60 """Changes to this property won't take effect until the current connection, if open, is closed and a new connection opened.""" 61 return self._connection_timeout 62 63 @connection_timeout.setter 64 def connection_timeout(self, timeout: float): 65 self._connection_timeout = timeout 66 67 @property 68 def detect_types(self) -> bool: 69 """Should use `detect_types=sqlite3.PARSE_DECLTYPES | sqlite3.PARSE_COLNAMES` when establishing a database connection. 70 71 Changes to this property won't take effect until the current connection, if open, is closed and a new connection opened. 72 """ 73 return self._detect_types 74 75 @detect_types.setter 76 def detect_types(self, should_detect: bool): 77 self._detect_types = should_detect 78 79 @property 80 def enforce_foreign_keys(self) -> bool: 81 return self._enforce_foreign_keys 82 83 @enforce_foreign_keys.setter 84 def enforce_foreign_keys(self, should_enforce: bool): 85 self._enforce_foreign_keys = should_enforce 86 self._set_foreign_key_enforcement() 87 88 @property 89 def name(self) -> str: 90 """The name of this database.""" 91 return self.path.stem 92 93 @property 94 def path(self) -> Pathier: 95 """The path to this database file.""" 96 return self._path 97 98 @path.setter 99 def path(self, new_path: Pathish): 100 """If `new_path` doesn't exist, it will be created (including parent folders).""" 101 self._path = Pathier(new_path) 102 if not self.path.exists(): 103 self.path.touch() 104 105 @property 106 def tables(self) -> list[str]: 107 """List of table names for this database.""" 108 return [ 109 table["name"] 110 for table in self.query( 111 "SELECT name FROM sqlite_Schema WHERE type = 'table' AND name NOT LIKE 'sqlite_%';" 112 ) 113 ] 114 115 def _logger_init(self, message_format: str, encoding: str): 116 """:param: `message_format`: `{` style format string.""" 117 self.logger = logging.getLogger(self.name) 118 if not self.logger.hasHandlers(): 119 handler = logging.FileHandler( 120 str(self.path).replace(".", "") + ".log", encoding=encoding 121 ) 122 handler.setFormatter( 123 logging.Formatter( 124 message_format, style="{", datefmt="%m/%d/%Y %I:%M:%S %p" 125 ) 126 ) 127 self.logger.addHandler(handler) 128 self.logger.setLevel(logging.INFO) 129 130 def _set_foreign_key_enforcement(self): 131 if self.connection: 132 self.connection.execute( 133 f"pragma foreign_keys = {int(self.enforce_foreign_keys)};" 134 ) 135 136 def add_column(self, table: str, column_def: str): 137 """Add a column to `table`. 138 139 `column_def` should be in the form `{column_name} {type_name} {constraint}`. 140 141 i.e. 142 >>> db = Databased() 143 >>> db.add_column("rides", "num_stops INTEGER NOT NULL DEFAULT 0")""" 144 self.query(f"ALTER TABLE {table} ADD {column_def};") 145 146 def close(self): 147 """Disconnect from the database. 148 149 Does not call `commit()` for you unless the `commit_on_close` property is set to `True`. 150 """ 151 if self.connection: 152 if self.commit_on_close: 153 self.commit() 154 self.connection.close() 155 self.connection = None 156 157 def commit(self): 158 """Commit state of database.""" 159 if self.connection: 160 self.connection.commit() 161 self.logger.info("Committed successfully.") 162 else: 163 raise RuntimeError( 164 "Databased.commit(): Can't commit db with no open connection." 165 ) 166 167 def connect(self): 168 """Connect to the database.""" 169 self.connection = sqlite3.connect( 170 self.path, 171 detect_types=sqlite3.PARSE_DECLTYPES | sqlite3.PARSE_COLNAMES 172 if self.detect_types 173 else 0, 174 timeout=self.connection_timeout, 175 ) 176 self._set_foreign_key_enforcement() 177 self.connection.row_factory = dict_factory 178 179 def count( 180 self, 181 table: str, 182 column: str = "*", 183 where: str | None = None, 184 distinct: bool = False, 185 ) -> int: 186 """Return number of matching rows in `table` table. 187 188 Equivalent to: 189 >>> SELECT COUNT({distinct} {column}) FROM {table} {where};""" 190 query = ( 191 f"SELECT COUNT( {('DISTINCT' if distinct else '')} {column}) FROM {table}" 192 ) 193 if where: 194 query += f" WHERE {where}" 195 query += ";" 196 return int(list(self.query(query)[0].values())[0]) 197 198 def create_table(self, table: str, *column_defs: str): 199 """Create a table if it doesn't exist. 200 201 #### :params: 202 203 `table`: Name of the table to create. 204 205 `column_defs`: Any number of column names and their definitions in proper Sqlite3 sytax. 206 i.e. `"column_name TEXT UNIQUE"` or `"column_name INTEGER PRIMARY KEY"` etc.""" 207 columns = ", ".join(column_defs) 208 result = self.query(f"CREATE TABLE IF NOT EXISTS {table} ({columns});") 209 self.logger.info(f"'{table}' table created.") 210 211 def delete(self, table: str, where: str | None = None) -> int: 212 """Delete rows from `table` that satisfy the given `where` clause. 213 214 If `where` is `None`, all rows will be deleted. 215 216 Returns the number of deleted rows. 217 218 e.g. 219 >>> db = Databased() 220 >>> db.delete("rides", "distance < 5 AND average_speed < 7")""" 221 try: 222 if where: 223 self.query(f"DELETE FROM {table} WHERE {where};") 224 else: 225 self.query(f"DELETE FROM {table};") 226 row_count = self.cursor.rowcount 227 self.logger.info( 228 f"Deleted {row_count} rows from '{table}' where '{where}'." 229 ) 230 return row_count 231 except Exception as e: 232 self.logger.exception( 233 f"Error deleting rows from '{table}' where '{where}'." 234 ) 235 raise e 236 237 def describe(self, table: str) -> list[dict]: 238 """Returns information about `table`.""" 239 return self.query(f"pragma table_info('{table}');") 240 241 def drop_column(self, table: str, column: str): 242 """Drop `column` from `table`.""" 243 self.query(f"ALTER TABLE {table} DROP {column};") 244 245 def drop_table(self, table: str) -> bool: 246 """Drop `table` from the database. 247 248 Returns `True` if successful, `False` if not.""" 249 try: 250 self.query(f"DROP TABLE {table};") 251 self.logger.info(f"Dropped table '{table}'.") 252 return True 253 except Exception as e: 254 print(f"{type(e).__name__}: {e}") 255 self.logger.error(f"Failed to drop table '{table}'.") 256 return False 257 258 def execute_script(self, path: Pathish, encoding: str = "utf-8") -> list[dict]: 259 """Execute sql script located at `path`.""" 260 if not self.connected: 261 self.connect() 262 assert self.connection 263 script = Pathier(path).read_text(encoding).replace("\n", " ") 264 return self.query(script) 265 266 def get_columns(self, table: str) -> list[str]: 267 """Returns a list of column names in `table`.""" 268 return [ 269 column["name"] for column in self.query(f"pragma table_info('{table}');") 270 ] 271 272 def insert( 273 self, table: str, columns: tuple[str, ...], values: list[tuple[Any, ...]] 274 ) -> int: 275 """Insert rows of `values` into `columns` of `table`. 276 277 Each `tuple` in `values` corresponds to an individual row that is to be inserted. 278 """ 279 max_row_count = 900 280 column_list = "(" + ", ".join(columns) + ")" 281 row_count = 0 282 for i in range(0, len(values), max_row_count): 283 chunk = values[i : i + max_row_count] 284 placeholder = ( 285 "(" + "),(".join((", ".join(("?" for _ in row)) for row in chunk)) + ")" 286 ) 287 logger_values = "\n".join( 288 ( 289 "'(" + ", ".join((str(value) for value in row)) + ")'" 290 for row in chunk 291 ) 292 ) 293 flattened_values = tuple((value for row in chunk for value in row)) 294 try: 295 self.query( 296 f"INSERT INTO {table} {column_list} VALUES {placeholder};", 297 flattened_values, 298 ) 299 self.logger.info( 300 f"Inserted into '{column_list}' columns of '{table}' table values \n{logger_values}." 301 ) 302 row_count += self.cursor.rowcount 303 except Exception as e: 304 self.logger.exception( 305 f"Error inserting into '{column_list}' columns of '{table}' table values \n{logger_values}." 306 ) 307 raise e 308 return row_count 309 310 def query(self, query_: str, parameters: tuple[Any, ...] = tuple()) -> list[dict]: 311 """Execute an SQL query and return the results. 312 313 Ensures that the database connection is opened before executing the command. 314 315 The cursor used to execute the query will be available through `self.cursor` until the next time `self.query()` is called. 316 """ 317 if not self.connected: 318 self.connect() 319 assert self.connection 320 self.cursor = self.connection.cursor() 321 self.cursor.execute(query_, parameters) 322 return self.cursor.fetchall() 323 324 def rename_column(self, table: str, column_to_rename: str, new_column_name: str): 325 """Rename a column in `table`.""" 326 self.query( 327 f"ALTER TABLE {table} RENAME {column_to_rename} TO {new_column_name};" 328 ) 329 330 def rename_table(self, table_to_rename: str, new_table_name: str): 331 """Rename a table.""" 332 self.query(f"ALTER TABLE {table_to_rename} RENAME TO {new_table_name};") 333 334 def select( 335 self, 336 table: str, 337 columns: list[str] = ["*"], 338 joins: list[str] | None = None, 339 where: str | None = None, 340 group_by: str | None = None, 341 having: str | None = None, 342 order_by: str | None = None, 343 limit: int | str | None = None, 344 ) -> list[dict]: 345 """Return rows for given criteria. 346 347 For complex queries, use the `databased.query()` method. 348 349 Parameters `where`, `group_by`, `having`, `order_by`, and `limit` should not have 350 their corresponding key word in their string, but should otherwise be valid SQL. 351 352 `joins` should contain their key word (`INNER JOIN`, `LEFT JOIN`) in addition to the rest of the sub-statement. 353 354 >>> Databased().select( 355 "bike_rides", 356 "id, date, distance, moving_time, AVG(distance/moving_time) as average_speed", 357 where="distance > 20", 358 order_by="distance", 359 desc=True, 360 limit=10 361 ) 362 executes the query: 363 >>> SELECT 364 id, date, distance, moving_time, AVG(distance/moving_time) as average_speed 365 FROM 366 bike_rides 367 WHERE 368 distance > 20 369 ORDER BY 370 distance DESC 371 Limit 10;""" 372 query = f"SELECT {', '.join(columns)} FROM {table}" 373 if joins: 374 query += f" {' '.join(joins)}" 375 if where: 376 query += f" WHERE {where}" 377 if group_by: 378 query += f" GROUP BY {group_by}" 379 if having: 380 query += f" HAVING {having}" 381 if order_by: 382 query += f" ORDER BY {order_by}" 383 if limit: 384 query += f" LIMIT {limit}" 385 query += ";" 386 rows = self.query(query) 387 return rows 388 389 @staticmethod 390 def to_grid(data: list[dict], shrink_to_terminal: bool = True) -> str: 391 """Returns a tabular grid from `data`. 392 393 If `shrink_to_terminal` is `True`, the column widths of the grid will be reduced to fit within the current terminal. 394 """ 395 return griddy(data, "keys", shrink_to_terminal) 396 397 def update( 398 self, table: str, column: str, value: Any, where: str | None = None 399 ) -> int: 400 """Update `column` of `table` to `value` for rows satisfying the conditions in `where`. 401 402 If `where` is `None` all rows will be updated. 403 404 Returns the number of updated rows. 405 406 e.g. 407 >>> db = Databased() 408 >>> db.update("rides", "elevation", 100, "elevation < 100")""" 409 try: 410 if where: 411 self.query(f"UPDATE {table} SET {column} = ? WHERE {where};", (value,)) 412 else: 413 self.query(f"UPDATE {table} SET {column} = ?;", (value,)) 414 row_count = self.cursor.rowcount 415 self.logger.info( 416 f"Updated {row_count} rows in '{table}' table to '{column}' = '{value}' where '{where}'." 417 ) 418 return row_count 419 except Exception as e: 420 self.logger.exception( 421 f"Failed to update rows in '{table}' table to '{column}' = '{value}' where '{where}'." 422 ) 423 raise e 424 425 def vacuum(self) -> int: 426 """Reduce disk size of database after row/table deletion. 427 428 Returns space freed up in bytes.""" 429 size = self.path.size 430 self.query("VACUUM;") 431 return size - self.path.size
SQLite3 wrapper.
18 def __init__( 19 self, 20 dbpath: Pathish = "db.sqlite3", 21 connection_timeout: float = 10, 22 detect_types: bool = True, 23 enforce_foreign_keys: bool = True, 24 commit_on_close: bool = True, 25 logger_encoding: str = "utf-8", 26 logger_message_format: str = "{levelname}|-|{asctime}|-|{message}", 27 ): 28 """ """ 29 self.path = dbpath 30 self.connection_timeout = connection_timeout 31 self.connection = None 32 self._logger_init(logger_message_format, logger_encoding) 33 self.detect_types = detect_types 34 self.commit_on_close = commit_on_close 35 self.enforce_foreign_keys = enforce_foreign_keys
If new_path
doesn't exist, it will be created (including parent folders).
Changes to this property won't take effect until the current connection, if open, is closed and a new connection opened.
Should use detect_types=sqlite3.PARSE_DECLTYPES | sqlite3.PARSE_COLNAMES
when establishing a database connection.
Changes to this property won't take effect until the current connection, if open, is closed and a new connection opened.
136 def add_column(self, table: str, column_def: str): 137 """Add a column to `table`. 138 139 `column_def` should be in the form `{column_name} {type_name} {constraint}`. 140 141 i.e. 142 >>> db = Databased() 143 >>> db.add_column("rides", "num_stops INTEGER NOT NULL DEFAULT 0")""" 144 self.query(f"ALTER TABLE {table} ADD {column_def};")
Add a column to table
.
column_def
should be in the form {column_name} {type_name} {constraint}
.
i.e.
>>> db = Databased()
>>> db.add_column("rides", "num_stops INTEGER NOT NULL DEFAULT 0")
146 def close(self): 147 """Disconnect from the database. 148 149 Does not call `commit()` for you unless the `commit_on_close` property is set to `True`. 150 """ 151 if self.connection: 152 if self.commit_on_close: 153 self.commit() 154 self.connection.close() 155 self.connection = None
Disconnect from the database.
Does not call commit()
for you unless the commit_on_close
property is set to True
.
157 def commit(self): 158 """Commit state of database.""" 159 if self.connection: 160 self.connection.commit() 161 self.logger.info("Committed successfully.") 162 else: 163 raise RuntimeError( 164 "Databased.commit(): Can't commit db with no open connection." 165 )
Commit state of database.
167 def connect(self): 168 """Connect to the database.""" 169 self.connection = sqlite3.connect( 170 self.path, 171 detect_types=sqlite3.PARSE_DECLTYPES | sqlite3.PARSE_COLNAMES 172 if self.detect_types 173 else 0, 174 timeout=self.connection_timeout, 175 ) 176 self._set_foreign_key_enforcement() 177 self.connection.row_factory = dict_factory
Connect to the database.
179 def count( 180 self, 181 table: str, 182 column: str = "*", 183 where: str | None = None, 184 distinct: bool = False, 185 ) -> int: 186 """Return number of matching rows in `table` table. 187 188 Equivalent to: 189 >>> SELECT COUNT({distinct} {column}) FROM {table} {where};""" 190 query = ( 191 f"SELECT COUNT( {('DISTINCT' if distinct else '')} {column}) FROM {table}" 192 ) 193 if where: 194 query += f" WHERE {where}" 195 query += ";" 196 return int(list(self.query(query)[0].values())[0])
Return number of matching rows in table
table.
Equivalent to:
>>> SELECT COUNT({distinct} {column}) FROM {table} {where};
198 def create_table(self, table: str, *column_defs: str): 199 """Create a table if it doesn't exist. 200 201 #### :params: 202 203 `table`: Name of the table to create. 204 205 `column_defs`: Any number of column names and their definitions in proper Sqlite3 sytax. 206 i.e. `"column_name TEXT UNIQUE"` or `"column_name INTEGER PRIMARY KEY"` etc.""" 207 columns = ", ".join(column_defs) 208 result = self.query(f"CREATE TABLE IF NOT EXISTS {table} ({columns});") 209 self.logger.info(f"'{table}' table created.")
Create a table if it doesn't exist.
:params:
table
: Name of the table to create.
column_defs
: Any number of column names and their definitions in proper Sqlite3 sytax.
i.e. "column_name TEXT UNIQUE"
or "column_name INTEGER PRIMARY KEY"
etc.
211 def delete(self, table: str, where: str | None = None) -> int: 212 """Delete rows from `table` that satisfy the given `where` clause. 213 214 If `where` is `None`, all rows will be deleted. 215 216 Returns the number of deleted rows. 217 218 e.g. 219 >>> db = Databased() 220 >>> db.delete("rides", "distance < 5 AND average_speed < 7")""" 221 try: 222 if where: 223 self.query(f"DELETE FROM {table} WHERE {where};") 224 else: 225 self.query(f"DELETE FROM {table};") 226 row_count = self.cursor.rowcount 227 self.logger.info( 228 f"Deleted {row_count} rows from '{table}' where '{where}'." 229 ) 230 return row_count 231 except Exception as e: 232 self.logger.exception( 233 f"Error deleting rows from '{table}' where '{where}'." 234 ) 235 raise e
Delete rows from table
that satisfy the given where
clause.
If where
is None
, all rows will be deleted.
Returns the number of deleted rows.
e.g.
>>> db = Databased()
>>> db.delete("rides", "distance < 5 AND average_speed < 7")
237 def describe(self, table: str) -> list[dict]: 238 """Returns information about `table`.""" 239 return self.query(f"pragma table_info('{table}');")
Returns information about table
.
241 def drop_column(self, table: str, column: str): 242 """Drop `column` from `table`.""" 243 self.query(f"ALTER TABLE {table} DROP {column};")
Drop column
from table
.
245 def drop_table(self, table: str) -> bool: 246 """Drop `table` from the database. 247 248 Returns `True` if successful, `False` if not.""" 249 try: 250 self.query(f"DROP TABLE {table};") 251 self.logger.info(f"Dropped table '{table}'.") 252 return True 253 except Exception as e: 254 print(f"{type(e).__name__}: {e}") 255 self.logger.error(f"Failed to drop table '{table}'.") 256 return False
Drop table
from the database.
Returns True
if successful, False
if not.
258 def execute_script(self, path: Pathish, encoding: str = "utf-8") -> list[dict]: 259 """Execute sql script located at `path`.""" 260 if not self.connected: 261 self.connect() 262 assert self.connection 263 script = Pathier(path).read_text(encoding).replace("\n", " ") 264 return self.query(script)
Execute sql script located at path
.
266 def get_columns(self, table: str) -> list[str]: 267 """Returns a list of column names in `table`.""" 268 return [ 269 column["name"] for column in self.query(f"pragma table_info('{table}');") 270 ]
Returns a list of column names in table
.
272 def insert( 273 self, table: str, columns: tuple[str, ...], values: list[tuple[Any, ...]] 274 ) -> int: 275 """Insert rows of `values` into `columns` of `table`. 276 277 Each `tuple` in `values` corresponds to an individual row that is to be inserted. 278 """ 279 max_row_count = 900 280 column_list = "(" + ", ".join(columns) + ")" 281 row_count = 0 282 for i in range(0, len(values), max_row_count): 283 chunk = values[i : i + max_row_count] 284 placeholder = ( 285 "(" + "),(".join((", ".join(("?" for _ in row)) for row in chunk)) + ")" 286 ) 287 logger_values = "\n".join( 288 ( 289 "'(" + ", ".join((str(value) for value in row)) + ")'" 290 for row in chunk 291 ) 292 ) 293 flattened_values = tuple((value for row in chunk for value in row)) 294 try: 295 self.query( 296 f"INSERT INTO {table} {column_list} VALUES {placeholder};", 297 flattened_values, 298 ) 299 self.logger.info( 300 f"Inserted into '{column_list}' columns of '{table}' table values \n{logger_values}." 301 ) 302 row_count += self.cursor.rowcount 303 except Exception as e: 304 self.logger.exception( 305 f"Error inserting into '{column_list}' columns of '{table}' table values \n{logger_values}." 306 ) 307 raise e 308 return row_count
Insert rows of values
into columns
of table
.
Each tuple
in values
corresponds to an individual row that is to be inserted.
310 def query(self, query_: str, parameters: tuple[Any, ...] = tuple()) -> list[dict]: 311 """Execute an SQL query and return the results. 312 313 Ensures that the database connection is opened before executing the command. 314 315 The cursor used to execute the query will be available through `self.cursor` until the next time `self.query()` is called. 316 """ 317 if not self.connected: 318 self.connect() 319 assert self.connection 320 self.cursor = self.connection.cursor() 321 self.cursor.execute(query_, parameters) 322 return self.cursor.fetchall()
Execute an SQL query and return the results.
Ensures that the database connection is opened before executing the command.
The cursor used to execute the query will be available through self.cursor
until the next time self.query()
is called.
324 def rename_column(self, table: str, column_to_rename: str, new_column_name: str): 325 """Rename a column in `table`.""" 326 self.query( 327 f"ALTER TABLE {table} RENAME {column_to_rename} TO {new_column_name};" 328 )
Rename a column in table
.
330 def rename_table(self, table_to_rename: str, new_table_name: str): 331 """Rename a table.""" 332 self.query(f"ALTER TABLE {table_to_rename} RENAME TO {new_table_name};")
Rename a table.
334 def select( 335 self, 336 table: str, 337 columns: list[str] = ["*"], 338 joins: list[str] | None = None, 339 where: str | None = None, 340 group_by: str | None = None, 341 having: str | None = None, 342 order_by: str | None = None, 343 limit: int | str | None = None, 344 ) -> list[dict]: 345 """Return rows for given criteria. 346 347 For complex queries, use the `databased.query()` method. 348 349 Parameters `where`, `group_by`, `having`, `order_by`, and `limit` should not have 350 their corresponding key word in their string, but should otherwise be valid SQL. 351 352 `joins` should contain their key word (`INNER JOIN`, `LEFT JOIN`) in addition to the rest of the sub-statement. 353 354 >>> Databased().select( 355 "bike_rides", 356 "id, date, distance, moving_time, AVG(distance/moving_time) as average_speed", 357 where="distance > 20", 358 order_by="distance", 359 desc=True, 360 limit=10 361 ) 362 executes the query: 363 >>> SELECT 364 id, date, distance, moving_time, AVG(distance/moving_time) as average_speed 365 FROM 366 bike_rides 367 WHERE 368 distance > 20 369 ORDER BY 370 distance DESC 371 Limit 10;""" 372 query = f"SELECT {', '.join(columns)} FROM {table}" 373 if joins: 374 query += f" {' '.join(joins)}" 375 if where: 376 query += f" WHERE {where}" 377 if group_by: 378 query += f" GROUP BY {group_by}" 379 if having: 380 query += f" HAVING {having}" 381 if order_by: 382 query += f" ORDER BY {order_by}" 383 if limit: 384 query += f" LIMIT {limit}" 385 query += ";" 386 rows = self.query(query) 387 return rows
Return rows for given criteria.
For complex queries, use the databased.query()
method.
Parameters where
, group_by
, having
, order_by
, and limit
should not have
their corresponding key word in their string, but should otherwise be valid SQL.
joins
should contain their key word (INNER JOIN
, LEFT JOIN
) in addition to the rest of the sub-statement.
>>> Databased().select(
"bike_rides",
"id, date, distance, moving_time, AVG(distance/moving_time) as average_speed",
where="distance > 20",
order_by="distance",
desc=True,
limit=10
)
executes the query:
>>> SELECT
id, date, distance, moving_time, AVG(distance/moving_time) as average_speed
FROM
bike_rides
WHERE
distance > 20
ORDER BY
distance DESC
Limit 10;
389 @staticmethod 390 def to_grid(data: list[dict], shrink_to_terminal: bool = True) -> str: 391 """Returns a tabular grid from `data`. 392 393 If `shrink_to_terminal` is `True`, the column widths of the grid will be reduced to fit within the current terminal. 394 """ 395 return griddy(data, "keys", shrink_to_terminal)
Returns a tabular grid from data
.
If shrink_to_terminal
is True
, the column widths of the grid will be reduced to fit within the current terminal.
397 def update( 398 self, table: str, column: str, value: Any, where: str | None = None 399 ) -> int: 400 """Update `column` of `table` to `value` for rows satisfying the conditions in `where`. 401 402 If `where` is `None` all rows will be updated. 403 404 Returns the number of updated rows. 405 406 e.g. 407 >>> db = Databased() 408 >>> db.update("rides", "elevation", 100, "elevation < 100")""" 409 try: 410 if where: 411 self.query(f"UPDATE {table} SET {column} = ? WHERE {where};", (value,)) 412 else: 413 self.query(f"UPDATE {table} SET {column} = ?;", (value,)) 414 row_count = self.cursor.rowcount 415 self.logger.info( 416 f"Updated {row_count} rows in '{table}' table to '{column}' = '{value}' where '{where}'." 417 ) 418 return row_count 419 except Exception as e: 420 self.logger.exception( 421 f"Failed to update rows in '{table}' table to '{column}' = '{value}' where '{where}'." 422 ) 423 raise e
Update column
of table
to value
for rows satisfying the conditions in where
.
If where
is None
all rows will be updated.
Returns the number of updated rows.
e.g.
>>> db = Databased()
>>> db.update("rides", "elevation", 100, "elevation < 100")
425 def vacuum(self) -> int: 426 """Reduce disk size of database after row/table deletion. 427 428 Returns space freed up in bytes.""" 429 size = self.path.size 430 self.query("VACUUM;") 431 return size - self.path.size
Reduce disk size of database after row/table deletion.
Returns space freed up in bytes.