Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1#!/usr/bin/env python 

2 

3# noinspection HttpUrlsUsage 

4""" 

5camcops_server/cc_modules/cc_config.py 

6 

7=============================================================================== 

8 

9 Copyright (C) 2012-2020 Rudolf Cardinal (rudolf@pobox.com). 

10 

11 This file is part of CamCOPS. 

12 

13 CamCOPS is free software: you can redistribute it and/or modify 

14 it under the terms of the GNU General Public License as published by 

15 the Free Software Foundation, either version 3 of the License, or 

16 (at your option) any later version. 

17 

18 CamCOPS is distributed in the hope that it will be useful, 

19 but WITHOUT ANY WARRANTY; without even the implied warranty of 

20 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

21 GNU General Public License for more details. 

22 

23 You should have received a copy of the GNU General Public License 

24 along with CamCOPS. If not, see <https://www.gnu.org/licenses/>. 

25 

26=============================================================================== 

27 

28**Read and represent a CamCOPS config file.** 

29 

30Also contains various types of demonstration config file (CamCOPS, but also 

31``supervisord``, Apache, etc.) and demonstration helper scripts (e.g. MySQL). 

32 

33There are CONDITIONAL AND IN-FUNCTION IMPORTS HERE; see below. This is to 

34minimize the number of modules loaded when this is used in the context of the 

35client-side database script, rather than the webview. 

36 

37Moreover, it should not use SQLAlchemy objects directly; see ``celery.py``. 

38 

39In particular, I tried hard to use a "database-unaware" (unbound) SQLAlchemy 

40ExportRecipient object. However, when the backend re-calls the config to get 

41its recipients, we get errors like: 

42 

43.. code-block:: none 

44 

45 [2018-12-25 00:56:00,118: ERROR/ForkPoolWorker-7] Task camcops_server.cc_modules.celery_tasks.export_to_recipient_backend[ab2e2691-c2fa-4821-b8cd-2cbeb86ddc8f] raised unexpected: DetachedInstanceError('Instance <ExportRecipient at 0x7febbeeea7b8> is not bound to a Session; attribute refresh operation cannot proceed',) 

46 Traceback (most recent call last): 

47 File "/home/rudolf/dev/venvs/camcops/lib/python3.6/site-packages/celery/app/trace.py", line 382, in trace_task 

48 R = retval = fun(*args, **kwargs) 

49 File "/home/rudolf/dev/venvs/camcops/lib/python3.6/site-packages/celery/app/trace.py", line 641, in __protected_call__ 

50 return self.run(*args, **kwargs) 

51 File "/home/rudolf/Documents/code/camcops/server/camcops_server/cc_modules/celery_tasks.py", line 103, in export_to_recipient_backend 

52 schedule_via_backend=False) 

53 File "/home/rudolf/Documents/code/camcops/server/camcops_server/cc_modules/cc_export.py", line 255, in export 

54 req, recipient_names=recipient_names, all_recipients=all_recipients) 

55 File "/home/rudolf/Documents/code/camcops/server/camcops_server/cc_modules/cc_config.py", line 1460, in get_export_recipients 

56 valid_names = set(r.recipient_name for r in recipients) 

57 File "/home/rudolf/Documents/code/camcops/server/camcops_server/cc_modules/cc_config.py", line 1460, in <genexpr> 

58 valid_names = set(r.recipient_name for r in recipients) 

59 File "/home/rudolf/dev/venvs/camcops/lib/python3.6/site-packages/sqlalchemy/orm/attributes.py", line 242, in __get__ 

60 return self.impl.get(instance_state(instance), dict_) 

61 File "/home/rudolf/dev/venvs/camcops/lib/python3.6/site-packages/sqlalchemy/orm/attributes.py", line 594, in get 

62 value = state._load_expired(state, passive) 

63 File "/home/rudolf/dev/venvs/camcops/lib/python3.6/site-packages/sqlalchemy/orm/state.py", line 608, in _load_expired 

64 self.manager.deferred_scalar_loader(self, toload) 

65 File "/home/rudolf/dev/venvs/camcops/lib/python3.6/site-packages/sqlalchemy/orm/loading.py", line 813, in load_scalar_attributes 

66 (state_str(state))) 

67 sqlalchemy.orm.exc.DetachedInstanceError: Instance <ExportRecipient at 0x7febbeeea7b8> is not bound to a Session; attribute refresh operation cannot proceed (Background on this error at: http://sqlalche.me/e/bhk3) 

68 

69""" # noqa 

70 

71import codecs 

72import collections 

73import configparser 

74import contextlib 

75import datetime 

76import os 

77import logging 

78import re 

79from typing import Any, Dict, Generator, List, Optional, Union 

80 

81from cardinal_pythonlib.configfiles import ( 

82 get_config_parameter, 

83 get_config_parameter_boolean, 

84 get_config_parameter_loglevel, 

85 get_config_parameter_multiline 

86) 

87from cardinal_pythonlib.docker import running_under_docker 

88from cardinal_pythonlib.fileops import relative_filename_within_dir 

89from cardinal_pythonlib.logs import BraceStyleAdapter 

90from cardinal_pythonlib.randomness import create_base64encoded_randomness 

91from cardinal_pythonlib.reprfunc import auto_repr 

92from cardinal_pythonlib.sqlalchemy.alembic_func import ( 

93 get_current_and_head_revision, 

94) 

95from cardinal_pythonlib.sqlalchemy.engine_func import ( 

96 is_sqlserver, 

97 is_sqlserver_2008_or_later, 

98) 

99from cardinal_pythonlib.sqlalchemy.logs import pre_disable_sqlalchemy_extra_echo_log # noqa 

100from cardinal_pythonlib.sqlalchemy.schema import get_table_names 

101from cardinal_pythonlib.sqlalchemy.session import get_safe_url_from_engine 

102from cardinal_pythonlib.wsgi.reverse_proxied_mw import ReverseProxiedMiddleware 

103import celery.schedules 

104from sqlalchemy.engine import create_engine 

105from sqlalchemy.engine.base import Engine 

106from sqlalchemy.orm import sessionmaker 

107from sqlalchemy.orm import Session as SqlASession 

108 

109from camcops_server.cc_modules.cc_baseconstants import ( 

110 ALEMBIC_BASE_DIR, 

111 ALEMBIC_CONFIG_FILENAME, 

112 ALEMBIC_VERSION_TABLE, 

113 ENVVAR_CONFIG_FILE, 

114 LINUX_DEFAULT_MATPLOTLIB_CACHE_DIR, 

115 ON_READTHEDOCS, 

116) 

117from camcops_server.cc_modules.cc_cache import cache_region_static, fkg 

118from camcops_server.cc_modules.cc_constants import ( 

119 CONFIG_FILE_EXPORT_SECTION, 

120 CONFIG_FILE_SERVER_SECTION, 

121 CONFIG_FILE_SITE_SECTION, 

122 ConfigDefaults, 

123 ConfigParamExportGeneral, 

124 ConfigParamExportRecipient, 

125 ConfigParamServer, 

126 ConfigParamSite, 

127 DockerConstants, 

128) 

129from camcops_server.cc_modules.cc_exportrecipientinfo import ( 

130 ExportRecipientInfo, 

131) 

132from camcops_server.cc_modules.cc_exception import raise_runtime_error 

133from camcops_server.cc_modules.cc_filename import ( 

134 PatientSpecElementForFilename, 

135) 

136from camcops_server.cc_modules.cc_language import POSSIBLE_LOCALES 

137from camcops_server.cc_modules.cc_pyramid import MASTER_ROUTE_CLIENT_API 

138from camcops_server.cc_modules.cc_snomed import ( 

139 get_all_task_snomed_concepts, 

140 get_icd9_snomed_concepts_from_xml, 

141 get_icd10_snomed_concepts_from_xml, 

142 SnomedConcept, 

143) 

144from camcops_server.cc_modules.cc_validators import ( 

145 validate_export_recipient_name, 

146 validate_group_name, 

147) 

148from camcops_server.cc_modules.cc_version_string import ( 

149 CAMCOPS_SERVER_VERSION_STRING, 

150) 

151 

152log = BraceStyleAdapter(logging.getLogger(__name__)) 

153 

154pre_disable_sqlalchemy_extra_echo_log() 

155 

156# ============================================================================= 

157# Constants 

158# ============================================================================= 

159 

160VALID_RECIPIENT_NAME_REGEX = r"^[\w_-]+$" 

161# ... because we'll use them for filenames, amongst other things 

162# https://stackoverflow.com/questions/10944438/ 

163# https://regexr.com/ 

164 

165# Windows paths: irrelevant, as Windows doesn't run supervisord 

166DEFAULT_LINUX_CAMCOPS_CONFIG = "/etc/camcops/camcops.conf" 

167DEFAULT_LINUX_CAMCOPS_BASE_DIR = "/usr/share/camcops" 

168DEFAULT_LINUX_CAMCOPS_VENV_DIR = os.path.join( 

169 DEFAULT_LINUX_CAMCOPS_BASE_DIR, "venv") 

170DEFAULT_LINUX_CAMCOPS_VENV_BIN_DIR = os.path.join( 

171 DEFAULT_LINUX_CAMCOPS_VENV_DIR, "bin") 

172DEFAULT_LINUX_CAMCOPS_EXECUTABLE = os.path.join( 

173 DEFAULT_LINUX_CAMCOPS_VENV_BIN_DIR, "camcops_server") 

174DEFAULT_LINUX_CAMCOPS_STATIC_DIR = os.path.join( 

175 DEFAULT_LINUX_CAMCOPS_VENV_DIR, 

176 "lib", "python3.6", "site-packages", "camcops_server", "static") 

177DEFAULT_LINUX_LOGDIR = "/var/log/supervisor" 

178DEFAULT_LINUX_USER = "www-data" # Ubuntu default 

179 

180 

181# ============================================================================= 

182# Helper functions 

183# ============================================================================= 

184 

185def warn_if_not_within_docker_dir(param_name: str, 

186 filespec: str, 

187 permit_cfg: bool = False, 

188 permit_venv: bool = False, 

189 permit_tmp: bool = False, 

190 param_contains_not_is: bool = False) -> None: 

191 """ 

192 If the specified filename isn't within a relevant directory that will be 

193 used by CamCOPS when operating within a Docker Compose application, warn 

194 the user. 

195 

196 Args: 

197 param_name: 

198 Name of the parameter in the CamCOPS config file. 

199 filespec: 

200 Filename (or filename-like thing) to check. 

201 permit_cfg: 

202 Permit the file to be in the configuration directory. 

203 permit_venv: 

204 Permit the file to be in the virtual environment directory. 

205 permit_tmp: 

206 Permit the file to be in the shared temporary space. 

207 param_contains_not_is: 

208 The parameter "contains", not "is", the filename. 

209 """ 

210 if not filespec: 

211 return 

212 is_phrase = "contains" if param_contains_not_is else "is" 

213 permitted_dirs = [] # type: List[str] 

214 if permit_cfg: 

215 permitted_dirs.append(DockerConstants.CONFIG_DIR) 

216 if permit_venv: 

217 permitted_dirs.append(DockerConstants.VENV_DIR) 

218 if permit_tmp: 

219 permitted_dirs.append(DockerConstants.TMP_DIR) 

220 ok = any( 

221 relative_filename_within_dir(filespec, d) 

222 for d in permitted_dirs 

223 ) 

224 if not ok: 

225 log.warning( 

226 f"Config parameter {param_name} {is_phrase} {filespec!r}, " 

227 f"which is not within the permitted Docker directories " 

228 f"{permitted_dirs!r}" 

229 ) 

230 

231 

232def warn_if_not_docker_value(param_name: str, 

233 actual_value: Any, 

234 required_value: Any) -> None: 

235 """ 

236 Warn the user if a parameter does not match the specific value required 

237 when operating under Docker. 

238 

239 Args: 

240 param_name: 

241 Name of the parameter in the CamCOPS config file. 

242 actual_value: 

243 Value in the config file. 

244 required_value: 

245 Value that should be used. 

246 """ 

247 if actual_value != required_value: 

248 log.warning( 

249 f"Config parameter {param_name} is {actual_value!r}, " 

250 f"but should be {required_value!r} when running inside Docker" 

251 ) 

252 

253 

254def warn_if_not_present(param_name: str, value: Any) -> None: 

255 """ 

256 Warn the user if a parameter is not set (None, or an empty string), for 

257 when operating under Docker. 

258 

259 Args: 

260 param_name: 

261 Name of the parameter in the CamCOPS config file. 

262 value: 

263 Value in the config file. 

264 """ 

265 if value is None or value == "": 

266 log.warning( 

267 f"Config parameter {param_name} is not specified, " 

268 f"but should be specified when running inside Docker" 

269 ) 

270 

271 

272# ============================================================================= 

273# Demo config 

274# ============================================================================= 

275 

276# Cosmetic demonstration constants: 

277DEFAULT_DB_READONLY_USER = 'QQQ_USERNAME_REPLACE_ME' 

278DEFAULT_DB_READONLY_PASSWORD = 'PPP_PASSWORD_REPLACE_ME' 

279DUMMY_INSTITUTION_URL = 'https://www.mydomain/' 

280 

281 

282def get_demo_config(for_docker: bool = False) -> str: 

283 """ 

284 Returns a demonstration config file based on the specified parameters. 

285 

286 Args: 

287 for_docker: 

288 Adjust defaults for the Docker environment. 

289 """ 

290 # ... 

291 # http://www.debian.org/doc/debian-policy/ch-opersys.html#s-writing-init 

292 # https://people.canonical.com/~cjwatson/ubuntu-policy/policy.html/ch-opersys.html # noqa 

293 session_cookie_secret = create_base64encoded_randomness(num_bytes=64) 

294 

295 cd = ConfigDefaults(docker=for_docker) 

296 return f""" 

297# Demonstration CamCOPS server configuration file. 

298# 

299# Created by CamCOPS server version {CAMCOPS_SERVER_VERSION_STRING}. 

300# See help at https://camcops.readthedocs.io/. 

301# 

302# Using defaults for Docker environment: {for_docker} 

303 

304# ============================================================================= 

305# CamCOPS site 

306# ============================================================================= 

307 

308[{CONFIG_FILE_SITE_SECTION}] 

309 

310# ----------------------------------------------------------------------------- 

311# Database connection 

312# ----------------------------------------------------------------------------- 

313 

314{ConfigParamSite.DB_URL} = {cd.demo_db_url} 

315{ConfigParamSite.DB_ECHO} = {cd.DB_ECHO} 

316 

317# ----------------------------------------------------------------------------- 

318# URLs and paths 

319# ----------------------------------------------------------------------------- 

320 

321{ConfigParamSite.LOCAL_INSTITUTION_URL} = {DUMMY_INSTITUTION_URL} 

322{ConfigParamSite.LOCAL_LOGO_FILE_ABSOLUTE} = {cd.LOCAL_LOGO_FILE_ABSOLUTE} 

323{ConfigParamSite.CAMCOPS_LOGO_FILE_ABSOLUTE} = {cd.CAMCOPS_LOGO_FILE_ABSOLUTE} 

324 

325{ConfigParamSite.EXTRA_STRING_FILES} = {cd.EXTRA_STRING_FILES} 

326{ConfigParamSite.RESTRICTED_TASKS} = 

327{ConfigParamSite.LANGUAGE} = {cd.LANGUAGE} 

328 

329{ConfigParamSite.SNOMED_TASK_XML_FILENAME} = 

330{ConfigParamSite.SNOMED_ICD9_XML_FILENAME} = 

331{ConfigParamSite.SNOMED_ICD10_XML_FILENAME} = 

332 

333{ConfigParamSite.WKHTMLTOPDF_FILENAME} = 

334 

335# ----------------------------------------------------------------------------- 

336# Login and session configuration 

337# ----------------------------------------------------------------------------- 

338 

339{ConfigParamSite.SESSION_COOKIE_SECRET} = camcops_autogenerated_secret_{session_cookie_secret} 

340{ConfigParamSite.SESSION_TIMEOUT_MINUTES} = {cd.SESSION_TIMEOUT_MINUTES} 

341{ConfigParamSite.PASSWORD_CHANGE_FREQUENCY_DAYS} = {cd.PASSWORD_CHANGE_FREQUENCY_DAYS} 

342{ConfigParamSite.LOCKOUT_THRESHOLD} = {cd.LOCKOUT_THRESHOLD} 

343{ConfigParamSite.LOCKOUT_DURATION_INCREMENT_MINUTES} = {cd.LOCKOUT_DURATION_INCREMENT_MINUTES} 

344{ConfigParamSite.DISABLE_PASSWORD_AUTOCOMPLETE} = {cd.DISABLE_PASSWORD_AUTOCOMPLETE} 

345 

346# ----------------------------------------------------------------------------- 

347# Suggested filenames for saving PDFs from the web view 

348# ----------------------------------------------------------------------------- 

349 

350{ConfigParamSite.PATIENT_SPEC_IF_ANONYMOUS} = {cd.PATIENT_SPEC_IF_ANONYMOUS} 

351{ConfigParamSite.PATIENT_SPEC} = {{{PatientSpecElementForFilename.SURNAME}}}_{{{PatientSpecElementForFilename.FORENAME}}}_{{{PatientSpecElementForFilename.ALLIDNUMS}}} 

352 

353{ConfigParamSite.TASK_FILENAME_SPEC} = CamCOPS_{{patient}}_{{created}}_{{tasktype}}-{{serverpk}}.{{filetype}} 

354{ConfigParamSite.TRACKER_FILENAME_SPEC} = CamCOPS_{{patient}}_{{now}}_tracker.{{filetype}} 

355{ConfigParamSite.CTV_FILENAME_SPEC} = CamCOPS_{{patient}}_{{now}}_clinicaltextview.{{filetype}} 

356 

357# ----------------------------------------------------------------------------- 

358# E-mail options 

359# ----------------------------------------------------------------------------- 

360 

361{ConfigParamSite.EMAIL_HOST} = mysmtpserver.mydomain 

362{ConfigParamSite.EMAIL_PORT} = {cd.EMAIL_PORT} 

363{ConfigParamSite.EMAIL_USE_TLS} = {cd.EMAIL_USE_TLS} 

364{ConfigParamSite.EMAIL_HOST_USERNAME} = myusername 

365{ConfigParamSite.EMAIL_HOST_PASSWORD} = mypassword 

366{ConfigParamSite.EMAIL_FROM} = CamCOPS computer <noreply@myinstitution.mydomain> 

367{ConfigParamSite.EMAIL_SENDER} = 

368{ConfigParamSite.EMAIL_REPLY_TO} = CamCOPS clinical administrator <admin@myinstitution.mydomain> 

369 

370# ----------------------------------------------------------------------------- 

371# User download options 

372# ----------------------------------------------------------------------------- 

373 

374{ConfigParamSite.PERMIT_IMMEDIATE_DOWNLOADS} = {cd.PERMIT_IMMEDIATE_DOWNLOADS} 

375{ConfigParamSite.USER_DOWNLOAD_DIR} = {cd.USER_DOWNLOAD_DIR} 

376{ConfigParamSite.USER_DOWNLOAD_FILE_LIFETIME_MIN} = {cd.USER_DOWNLOAD_FILE_LIFETIME_MIN} 

377{ConfigParamSite.USER_DOWNLOAD_MAX_SPACE_MB} = {cd.USER_DOWNLOAD_MAX_SPACE_MB} 

378 

379# ----------------------------------------------------------------------------- 

380# Debugging options 

381# ----------------------------------------------------------------------------- 

382 

383{ConfigParamSite.WEBVIEW_LOGLEVEL} = {cd.WEBVIEW_LOGLEVEL_TEXTFORMAT} 

384{ConfigParamSite.CLIENT_API_LOGLEVEL} = {cd.CLIENT_API_LOGLEVEL_TEXTFORMAT} 

385{ConfigParamSite.ALLOW_INSECURE_COOKIES} = {cd.ALLOW_INSECURE_COOKIES} 

386 

387 

388# ============================================================================= 

389# Web server options 

390# ============================================================================= 

391 

392[{CONFIG_FILE_SERVER_SECTION}] 

393 

394# ----------------------------------------------------------------------------- 

395# Common web server options 

396# ----------------------------------------------------------------------------- 

397 

398{ConfigParamServer.HOST} = {cd.HOST} 

399{ConfigParamServer.PORT} = {cd.PORT} 

400{ConfigParamServer.UNIX_DOMAIN_SOCKET} = 

401 

402# If you host CamCOPS behind Apache, it’s likely that you’ll want Apache to 

403# handle HTTPS and CamCOPS to operate unencrypted behind a reverse proxy, in 

404# which case don’t set SSL_CERTIFICATE or SSL_PRIVATE_KEY. 

405{ConfigParamServer.SSL_CERTIFICATE} = 

406{ConfigParamServer.SSL_PRIVATE_KEY} = 

407{ConfigParamServer.STATIC_CACHE_DURATION_S} = {cd.STATIC_CACHE_DURATION_S} 

408 

409# ----------------------------------------------------------------------------- 

410# WSGI options 

411# ----------------------------------------------------------------------------- 

412 

413{ConfigParamServer.DEBUG_REVERSE_PROXY} = {cd.DEBUG_REVERSE_PROXY} 

414{ConfigParamServer.DEBUG_TOOLBAR} = {cd.DEBUG_TOOLBAR} 

415{ConfigParamServer.SHOW_REQUESTS} = {cd.SHOW_REQUESTS} 

416{ConfigParamServer.SHOW_REQUEST_IMMEDIATELY} = {cd.SHOW_REQUEST_IMMEDIATELY} 

417{ConfigParamServer.SHOW_RESPONSE} = {cd.SHOW_RESPONSE} 

418{ConfigParamServer.SHOW_TIMING} = {cd.SHOW_TIMING} 

419{ConfigParamServer.PROXY_HTTP_HOST} = 

420{ConfigParamServer.PROXY_REMOTE_ADDR} = 

421{ConfigParamServer.PROXY_REWRITE_PATH_INFO} = {cd.PROXY_REWRITE_PATH_INFO} 

422{ConfigParamServer.PROXY_SCRIPT_NAME} = 

423{ConfigParamServer.PROXY_SERVER_NAME} = 

424{ConfigParamServer.PROXY_SERVER_PORT} = 

425{ConfigParamServer.PROXY_URL_SCHEME} = 

426{ConfigParamServer.TRUSTED_PROXY_HEADERS} = 

427 HTTP_X_FORWARDED_HOST 

428 HTTP_X_FORWARDED_SERVER 

429 HTTP_X_FORWARDED_PORT 

430 HTTP_X_FORWARDED_PROTO 

431 HTTP_X_FORWARDED_FOR 

432 HTTP_X_SCRIPT_NAME 

433 

434# ----------------------------------------------------------------------------- 

435# CherryPy options 

436# ----------------------------------------------------------------------------- 

437 

438{ConfigParamServer.CHERRYPY_SERVER_NAME} = {cd.CHERRYPY_SERVER_NAME} 

439{ConfigParamServer.CHERRYPY_THREADS_START} = {cd.CHERRYPY_THREADS_START} 

440{ConfigParamServer.CHERRYPY_THREADS_MAX} = {cd.CHERRYPY_THREADS_MAX} 

441{ConfigParamServer.CHERRYPY_LOG_SCREEN} = {cd.CHERRYPY_LOG_SCREEN} 

442{ConfigParamServer.CHERRYPY_ROOT_PATH} = {cd.CHERRYPY_ROOT_PATH} 

443 

444# ----------------------------------------------------------------------------- 

445# Gunicorn options 

446# ----------------------------------------------------------------------------- 

447 

448{ConfigParamServer.GUNICORN_NUM_WORKERS} = {cd.GUNICORN_NUM_WORKERS} 

449{ConfigParamServer.GUNICORN_DEBUG_RELOAD} = {cd.GUNICORN_DEBUG_RELOAD} 

450{ConfigParamServer.GUNICORN_TIMEOUT_S} = {cd.GUNICORN_TIMEOUT_S} 

451{ConfigParamServer.DEBUG_SHOW_GUNICORN_OPTIONS} = {cd.DEBUG_SHOW_GUNICORN_OPTIONS} 

452 

453 

454# ============================================================================= 

455# Export options 

456# ============================================================================= 

457 

458[{CONFIG_FILE_EXPORT_SECTION}] 

459 

460{ConfigParamExportGeneral.CELERY_BEAT_EXTRA_ARGS} = 

461{ConfigParamExportGeneral.CELERY_BEAT_SCHEDULE_DATABASE} = {cd.CELERY_BEAT_SCHEDULE_DATABASE} 

462{ConfigParamExportGeneral.CELERY_BROKER_URL} = {cd.CELERY_BROKER_URL} 

463{ConfigParamExportGeneral.CELERY_WORKER_EXTRA_ARGS} = 

464 --maxtasksperchild=1000 

465{ConfigParamExportGeneral.CELERY_EXPORT_TASK_RATE_LIMIT} = 100/m 

466{ConfigParamExportGeneral.EXPORT_LOCKDIR} = {cd.EXPORT_LOCKDIR} 

467 

468{ConfigParamExportGeneral.RECIPIENTS} = 

469 

470{ConfigParamExportGeneral.SCHEDULE_TIMEZONE} = {cd.SCHEDULE_TIMEZONE} 

471{ConfigParamExportGeneral.SCHEDULE} = 

472 

473 

474# ============================================================================= 

475# Details for each export recipient 

476# ============================================================================= 

477 

478# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ 

479# Example recipient 

480# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ 

481 # Example (disabled because it's not in the {ConfigParamExportGeneral.RECIPIENTS} list above) 

482 

483[recipient:recipient_A] 

484 

485 # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ 

486 # How to export 

487 # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ 

488 

489{ConfigParamExportRecipient.TRANSMISSION_METHOD} = hl7 

490{ConfigParamExportRecipient.PUSH} = true 

491{ConfigParamExportRecipient.TASK_FORMAT} = pdf 

492{ConfigParamExportRecipient.XML_FIELD_COMMENTS} = {cd.XML_FIELD_COMMENTS} 

493 

494 # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ 

495 # What to export 

496 # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ 

497 

498{ConfigParamExportRecipient.ALL_GROUPS} = false 

499{ConfigParamExportRecipient.GROUPS} = 

500 myfirstgroup 

501 mysecondgroup 

502{ConfigParamExportRecipient.TASKS} = 

503 

504{ConfigParamExportRecipient.START_DATETIME_UTC} = 

505{ConfigParamExportRecipient.END_DATETIME_UTC} = 

506{ConfigParamExportRecipient.FINALIZED_ONLY} = {cd.FINALIZED_ONLY} 

507{ConfigParamExportRecipient.INCLUDE_ANONYMOUS} = {cd.INCLUDE_ANONYMOUS} 

508{ConfigParamExportRecipient.PRIMARY_IDNUM} = 1 

509{ConfigParamExportRecipient.REQUIRE_PRIMARY_IDNUM_MANDATORY_IN_POLICY} = {cd.REQUIRE_PRIMARY_IDNUM_MANDATORY_IN_POLICY} 

510 

511 # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ 

512 # Options applicable to database exports 

513 # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ 

514 

515{ConfigParamExportRecipient.DB_URL} = some_sqlalchemy_url 

516{ConfigParamExportRecipient.DB_ECHO} = {cd.DB_ECHO} 

517{ConfigParamExportRecipient.DB_INCLUDE_BLOBS} = {cd.DB_INCLUDE_BLOBS} 

518{ConfigParamExportRecipient.DB_ADD_SUMMARIES} = {cd.DB_ADD_SUMMARIES} 

519{ConfigParamExportRecipient.DB_PATIENT_ID_PER_ROW} = {cd.DB_PATIENT_ID_PER_ROW} 

520 

521 # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ 

522 # Options applicable to e-mail exports 

523 # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ 

524 

525{ConfigParamExportRecipient.EMAIL_TO} = 

526 Perinatal Psychiatry Admin <perinatal@myinstitution.mydomain> 

527 

528{ConfigParamExportRecipient.EMAIL_CC} = 

529 Dr Alice Bradford <alice.bradford@myinstitution.mydomain> 

530 Dr Charles Dogfoot <charles.dogfoot@myinstitution.mydomain> 

531 

532{ConfigParamExportRecipient.EMAIL_BCC} = 

533 superuser <root@myinstitution.mydomain> 

534 

535{ConfigParamExportRecipient.EMAIL_PATIENT_SPEC_IF_ANONYMOUS} = anonymous 

536{ConfigParamExportRecipient.EMAIL_PATIENT_SPEC} = {{{PatientSpecElementForFilename.SURNAME}}}, {{{PatientSpecElementForFilename.FORENAME}}}, {{{PatientSpecElementForFilename.ALLIDNUMS}}} 

537{ConfigParamExportRecipient.EMAIL_SUBJECT} = CamCOPS task for {{patient}}, created {{created}}: {{tasktype}}, PK {{serverpk}} 

538{ConfigParamExportRecipient.EMAIL_BODY_IS_HTML} = false 

539{ConfigParamExportRecipient.EMAIL_BODY} = 

540 Please find attached a new CamCOPS task for manual filing to the electronic 

541 patient record of 

542 

543 {{patient}} 

544 

545 Task type: {{tasktype}} 

546 Created: {{created}} 

547 CamCOPS server primary key: {{serverpk}} 

548 

549 Yours faithfully, 

550 

551 The CamCOPS computer. 

552 

553{ConfigParamExportRecipient.EMAIL_KEEP_MESSAGE} = {cd.HL7_KEEP_MESSAGE} 

554 

555 # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ 

556 # Options applicable to HL7 

557 # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ 

558 

559{ConfigParamExportRecipient.HL7_HOST} = myhl7server.mydomain 

560{ConfigParamExportRecipient.HL7_PORT} = {cd.HL7_PORT} 

561{ConfigParamExportRecipient.HL7_PING_FIRST} = {cd.HL7_PING_FIRST} 

562{ConfigParamExportRecipient.HL7_NETWORK_TIMEOUT_MS} = {cd.HL7_NETWORK_TIMEOUT_MS} 

563{ConfigParamExportRecipient.HL7_KEEP_MESSAGE} = {cd.HL7_KEEP_MESSAGE} 

564{ConfigParamExportRecipient.HL7_KEEP_REPLY} = {cd.HL7_KEEP_REPLY} 

565{ConfigParamExportRecipient.HL7_DEBUG_DIVERT_TO_FILE} = {cd.HL7_DEBUG_DIVERT_TO_FILE} 

566{ConfigParamExportRecipient.HL7_DEBUG_TREAT_DIVERTED_AS_SENT} = {cd.HL7_DEBUG_TREAT_DIVERTED_AS_SENT} 

567 

568 # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ 

569 # Options applicable to file transfers/attachments 

570 # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ 

571 

572{ConfigParamExportRecipient.FILE_PATIENT_SPEC} = {{surname}}_{{forename}}_{{idshortdesc1}}{{idnum1}} 

573{ConfigParamExportRecipient.FILE_PATIENT_SPEC_IF_ANONYMOUS} = {cd.FILE_PATIENT_SPEC_IF_ANONYMOUS} 

574{ConfigParamExportRecipient.FILE_FILENAME_SPEC} = /my_nfs_mount/mypath/CamCOPS_{{patient}}_{{created}}_{{tasktype}}-{{serverpk}}.{{filetype}} 

575{ConfigParamExportRecipient.FILE_MAKE_DIRECTORY} = {cd.FILE_MAKE_DIRECTORY} 

576{ConfigParamExportRecipient.FILE_OVERWRITE_FILES} = {cd.FILE_OVERWRITE_FILES} 

577{ConfigParamExportRecipient.FILE_EXPORT_RIO_METADATA} = {cd.FILE_EXPORT_RIO_METADATA} 

578{ConfigParamExportRecipient.FILE_SCRIPT_AFTER_EXPORT} = 

579 

580 # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ 

581 # Extra options for RiO metadata for file-based export 

582 # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ 

583 

584{ConfigParamExportRecipient.RIO_IDNUM} = 2 

585{ConfigParamExportRecipient.RIO_UPLOADING_USER} = CamCOPS 

586{ConfigParamExportRecipient.RIO_DOCUMENT_TYPE} = CC 

587 

588 # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ 

589 # Extra options for REDCap export 

590 # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ 

591 

592{ConfigParamExportRecipient.REDCAP_API_URL} = https://domain.of.redcap.server/api/ 

593{ConfigParamExportRecipient.REDCAP_API_KEY} = myapikey 

594{ConfigParamExportRecipient.REDCAP_FIELDMAP_FILENAME} = /location/of/fieldmap.xml 

595 

596 """.strip() # noqa 

597 

598 

599# ============================================================================= 

600# Demo configuration files, other than the CamCOPS config file itself 

601# ============================================================================= 

602 

603DEFAULT_SOCKET_FILENAME = "/run/camcops/camcops.socket" 

604 

605 

606def get_demo_supervisor_config() -> str: 

607 """ 

608 Returns a demonstration ``supervisord`` config file based on the 

609 specified parameters. 

610 """ 

611 redirect_stderr = "true" 

612 autostart = "true" 

613 autorestart = "true" 

614 startsecs = "30" 

615 stopwaitsecs = "60" 

616 return f""" 

617# ============================================================================= 

618# Demonstration 'supervisor' (supervisord) config file for CamCOPS. 

619# Created by CamCOPS version {CAMCOPS_SERVER_VERSION_STRING}. 

620# ============================================================================= 

621# See https://camcops.readthedocs.io/en/latest/administrator/server_configuration.html#start-camcops 

622 

623[program:camcops_server] 

624 

625command = {DEFAULT_LINUX_CAMCOPS_EXECUTABLE} serve_gunicorn 

626 --config {DEFAULT_LINUX_CAMCOPS_CONFIG} 

627 

628directory = {DEFAULT_LINUX_CAMCOPS_BASE_DIR} 

629environment = MPLCONFIGDIR="{LINUX_DEFAULT_MATPLOTLIB_CACHE_DIR}" 

630user = {DEFAULT_LINUX_USER} 

631stdout_logfile = {DEFAULT_LINUX_LOGDIR}/camcops_server.log 

632redirect_stderr = {redirect_stderr} 

633autostart = {autostart} 

634autorestart = {autorestart} 

635startsecs = {startsecs} 

636stopwaitsecs = {stopwaitsecs} 

637 

638[program:camcops_workers] 

639 

640command = {DEFAULT_LINUX_CAMCOPS_EXECUTABLE} launch_workers 

641 --config {DEFAULT_LINUX_CAMCOPS_CONFIG} 

642 

643directory = {DEFAULT_LINUX_CAMCOPS_BASE_DIR} 

644environment = MPLCONFIGDIR="{LINUX_DEFAULT_MATPLOTLIB_CACHE_DIR}" 

645user = {DEFAULT_LINUX_USER} 

646stdout_logfile = {DEFAULT_LINUX_LOGDIR}/camcops_workers.log 

647redirect_stderr = {redirect_stderr} 

648autostart = {autostart} 

649autorestart = {autorestart} 

650startsecs = {startsecs} 

651stopwaitsecs = {stopwaitsecs} 

652 

653[program:camcops_scheduler] 

654 

655command = {DEFAULT_LINUX_CAMCOPS_EXECUTABLE} launch_scheduler 

656 --config {DEFAULT_LINUX_CAMCOPS_CONFIG} 

657 

658directory = {DEFAULT_LINUX_CAMCOPS_BASE_DIR} 

659environment = MPLCONFIGDIR="{LINUX_DEFAULT_MATPLOTLIB_CACHE_DIR}" 

660user = {DEFAULT_LINUX_USER} 

661stdout_logfile = {DEFAULT_LINUX_LOGDIR}/camcops_scheduler.log 

662redirect_stderr = {redirect_stderr} 

663autostart = {autostart} 

664autorestart = {autorestart} 

665startsecs = {startsecs} 

666stopwaitsecs = {stopwaitsecs} 

667 

668[group:camcops] 

669 

670programs = camcops_server, camcops_workers, camcops_scheduler 

671 

672 """.strip() # noqa 

673 

674 

675def get_demo_apache_config( 

676 rootpath: str = "camcops", # no slash 

677 specimen_internal_port: int = None, 

678 specimen_socket_file: str = DEFAULT_SOCKET_FILENAME) -> str: 

679 """ 

680 Returns a demo Apache HTTPD config file section applicable to CamCOPS. 

681 """ 

682 cd = ConfigDefaults() 

683 specimen_internal_port = specimen_internal_port or cd.PORT 

684 urlbase = "/" + rootpath 

685 # noinspection HttpUrlsUsage 

686 return f""" 

687# Demonstration Apache config file section for CamCOPS. 

688# Created by CamCOPS version {CAMCOPS_SERVER_VERSION_STRING}. 

689# 

690# Under Ubuntu, the Apache config will be somewhere in /etc/apache2/ 

691# Under CentOS, the Apache config will be somewhere in /etc/httpd/ 

692# 

693# This section should go within the <VirtualHost> directive for the secure 

694# (SSL, HTTPS) part of the web site. 

695 

696<VirtualHost *:443> 

697 # ... 

698 

699 # ========================================================================= 

700 # CamCOPS 

701 # ========================================================================= 

702 # Apache operates on the principle that the first match wins. So, if we 

703 # want to serve CamCOPS but then override some of its URLs to serve static 

704 # files faster, we define the static stuff first. 

705 

706 # --------------------------------------------------------------------- 

707 # 1. Serve static files 

708 # --------------------------------------------------------------------- 

709 # a) offer them at the appropriate URL 

710 # b) provide permission 

711 # c) disable ProxyPass for static files 

712 

713 # CHANGE THIS: aim the alias at your own institutional logo. 

714 

715 Alias {urlbase}/static/logo_local.png {DEFAULT_LINUX_CAMCOPS_STATIC_DIR}/logo_local.png 

716 

717 # We move from more specific to less specific aliases; the first match 

718 # takes precedence. (Apache will warn about conflicting aliases if 

719 # specified in a wrong, less-to-more-specific, order.) 

720 

721 Alias {urlbase}/static/ {DEFAULT_LINUX_CAMCOPS_STATIC_DIR}/ 

722 

723 <Directory {DEFAULT_LINUX_CAMCOPS_STATIC_DIR}> 

724 Require all granted 

725 

726 # ... for old Apache versions (e.g. 2.2), use instead: 

727 # Order allow,deny 

728 # Allow from all 

729 </Directory> 

730 

731 # Don't ProxyPass the static files; we'll serve them via Apache. 

732 

733 ProxyPassMatch ^{urlbase}/static/ ! 

734 

735 # --------------------------------------------------------------------- 

736 # 2. Proxy requests to the CamCOPS web server and back; allow access 

737 # --------------------------------------------------------------------- 

738 # ... either via an internal TCP/IP port (e.g. 1024 or higher, and NOT 

739 # accessible to users); 

740 # ... or, better, via a Unix socket, e.g. {specimen_socket_file} 

741 # 

742 # NOTES 

743 # 

744 # - When you ProxyPass {urlbase}, you should browse to 

745 # 

746 # https://YOURSITE{urlbase} 

747 # 

748 # and point your tablet devices to 

749 # 

750 # https://YOURSITE{urlbase}{MASTER_ROUTE_CLIENT_API} 

751 # 

752 # - Don't specify trailing slashes for the ProxyPass and 

753 # ProxyPassReverse directives. 

754 # If you do, http://host/camcops will fail though 

755 # http://host/camcops/ will succeed. 

756 # 

757 # - An alternative fix is to enable mod_rewrite (e.g. sudo a2enmod 

758 # rewrite), then add these commands: 

759 # 

760 # RewriteEngine on 

761 # RewriteRule ^/{rootpath}$ {rootpath}/ [L,R=301] 

762 # 

763 # which will redirect requests without the trailing slash to a 

764 # version with the trailing slash. 

765 # 

766 # - Ensure that you put the CORRECT PROTOCOL (http, https) in the rules 

767 # below. 

768 # 

769 # - For ProxyPass options, see https://httpd.apache.org/docs/2.2/mod/mod_proxy.html#proxypass 

770 # 

771 # - Include "retry=0" to stop Apache disabling the connection for 

772 # while on failure. 

773 # - Consider adding a "timeout=<seconds>" option if the back-end is 

774 # slow and causing timeouts. 

775 # 

776 # - CamCOPS MUST BE TOLD about its location and protocol, because that 

777 # information is critical for synthesizing URLs, but is stripped out 

778 # by the reverse proxy system. There are two ways: 

779 # 

780 # (i) specifying headers or WSGI environment variables, such as 

781 # the HTTP(S) headers X-Forwarded-Proto and X-Script-Name below 

782 # (and telling CamCOPS to trust them via its 

783 # TRUSTED_PROXY_HEADERS setting); 

784 # 

785 # (ii) specifying other options to "camcops_server", including 

786 # PROXY_SCRIPT_NAME, PROXY_URL_SCHEME; see the help for the 

787 # CamCOPS config. 

788 # 

789 # So: 

790 # 

791 # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ 

792 # (a) Reverse proxy 

793 # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ 

794 # 

795 # ##################################################################### 

796 # PORT METHOD 

797 # ##################################################################### 

798 # Note the use of "http" (reflecting the backend), not https (like the 

799 # front end). 

800 

801 # ProxyPass {urlbase} http://127.0.0.1:{specimen_internal_port} retry=0 timeout=300 

802 # ProxyPassReverse {urlbase} http://127.0.0.1:{specimen_internal_port} 

803 

804 # ##################################################################### 

805 # UNIX SOCKET METHOD (Apache 2.4.9 and higher) 

806 # ##################################################################### 

807 # This requires Apache 2.4.9, and passes after the '|' character a URL 

808 # that determines the Host: value of the request; see 

809 # ://httpd.apache.org/docs/trunk/mod/mod_proxy.html#proxypass 

810 # 

811 # The general syntax is: 

812 # 

813 # ProxyPass /URL_USER_SEES unix:SOCKETFILE|PROTOCOL://HOST/EXTRA_URL_FOR_BACKEND retry=0 

814 # 

815 # Note that: 

816 # 

817 # - the protocol should be http, not https (Apache deals with the 

818 # HTTPS part and passes HTTP on) 

819 # - the EXTRA_URL_FOR_BACKEND needs to be (a) unique for each 

820 # instance or Apache will use a single worker for multiple 

821 # instances, and (b) blank for the backend's benefit. Since those 

822 # two conflict when there's >1 instance, there's a problem. 

823 # - Normally, HOST is given as localhost. It may be that this problem 

824 # is solved by using a dummy unique value for HOST: 

825 # https://bz.apache.org/bugzilla/show_bug.cgi?id=54101#c1 

826 # 

827 # If your Apache version is too old, you will get the error 

828 # 

829 # "AH00526: Syntax error on line 56 of /etc/apache2/sites-enabled/SOMETHING: 

830 # ProxyPass URL must be absolute!" 

831 # 

832 # If you get this error: 

833 # 

834 # AH01146: Ignoring parameter 'retry=0' for worker 'unix:/tmp/.camcops_gunicorn.sock|https://localhost' because of worker sharing 

835 # https://wiki.apache.org/httpd/ListOfErrors 

836 # 

837 # ... then your URLs are overlapping and should be redone or sorted; 

838 # see http://httpd.apache.org/docs/2.4/mod/mod_proxy.html#workers 

839 # 

840 # The part that must be unique for each instance, with no part a 

841 # leading substring of any other, is THIS_BIT in: 

842 # 

843 # ProxyPass /URL_USER_SEES unix:SOCKETFILE|http://localhost/THIS_BIT retry=0 

844 # 

845 # If you get an error like this: 

846 # 

847 # AH01144: No protocol handler was valid for the URL /SOMEWHERE. If you are using a DSO version of mod_proxy, make sure the proxy submodules are included in the configuration using LoadModule. 

848 # 

849 # Then do this: 

850 # 

851 # sudo a2enmod proxy proxy_http 

852 # sudo apache2ctl restart 

853 # 

854 # If you get an error like this: 

855 # 

856 # ... [proxy_http:error] [pid 32747] (103)Software caused connection abort: [client 109.151.49.173:56898] AH01102: error reading status line from remote server httpd-UDS:0 

857 # [proxy:error] [pid 32747] [client 109.151.49.173:56898] AH00898: Error reading from remote server returned by /camcops_bruhl/webview 

858 # 

859 # then check you are specifying http://, not https://, in the ProxyPass 

860 # 

861 # Other information sources: 

862 # 

863 # - https://emptyhammock.com/projects/info/pyweb/webconfig.html 

864 

865 ProxyPass {urlbase} unix:{specimen_socket_file}|http://dummy1 retry=0 timeout=300 

866 ProxyPassReverse {urlbase} unix:{specimen_socket_file}|http://dummy1 

867 

868 # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ 

869 # (b) Allow proxy over SSL. 

870 # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ 

871 # Without this, you will get errors like: 

872 # ... SSL Proxy requested for wombat:443 but not enabled [Hint: SSLProxyEngine] 

873 # ... failed to enable ssl support for 0.0.0.0:0 (httpd-UDS) 

874 

875 SSLProxyEngine on 

876 

877 <Location /camcops> 

878 

879 # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ 

880 # (c) Allow access 

881 # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ 

882 

883 Require all granted 

884 

885 # ... for old Apache versions (e.g. 2.2), use instead: 

886 # 

887 # Order allow,deny 

888 # Allow from all 

889 

890 # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ 

891 # (d) Tell the proxied application that we are using HTTPS, and 

892 # where the application is installed 

893 # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ 

894 # ... https://stackoverflow.com/questions/16042647 

895 # 

896 # Enable mod_headers (e.g. "sudo a2enmod headers") and set: 

897 

898 RequestHeader set X-Forwarded-Proto https 

899 RequestHeader set X-Script-Name {urlbase} 

900 

901 # ... then ensure the TRUSTED_PROXY_HEADERS setting in the CamCOPS 

902 # config file includes: 

903 # 

904 # HTTP_X_FORWARDED_HOST 

905 # HTTP_X_FORWARDED_SERVER 

906 # HTTP_X_FORWARDED_PORT 

907 # HTTP_X_FORWARDED_PROTO 

908 # HTTP_X_SCRIPT_NAME 

909 # 

910 # (X-Forwarded-For, X-Forwarded-Host, and X-Forwarded-Server are 

911 # supplied by Apache automatically.) 

912 

913 </Location> 

914 

915 #========================================================================== 

916 # SSL security (for HTTPS) 

917 #========================================================================== 

918 

919 # You will also need to install your SSL certificate; see the 

920 # instructions that came with it. You get a certificate by creating a 

921 # certificate signing request (CSR). You enter some details about your 

922 # site, and a software tool makes (1) a private key, which you keep 

923 # utterly private, and (2) a CSR, which you send to a Certificate 

924 # Authority (CA) for signing. They send back a signed certificate, and 

925 # a chain of certificates leading from yours to a trusted root CA. 

926 # 

927 # You can create your own (a 'snake-oil' certificate), but your tablets 

928 # and browsers will not trust it, so this is a bad idea. 

929 # 

930 # Once you have your certificate: edit and uncomment these lines: 

931 

932 # SSLEngine on 

933 

934 # SSLCertificateKeyFile /etc/ssl/private/my.private.key 

935 

936 # ... a private file that you made before creating the certificate 

937 # request, and NEVER GAVE TO ANYBODY, and NEVER WILL (or your 

938 # security is broken and you need a new certificate). 

939 

940 # SSLCertificateFile /etc/ssl/certs/my.public.cert 

941 

942 # ... signed and supplied to you by the certificate authority (CA), 

943 # from the public certificate you sent to them. 

944 

945 # SSLCertificateChainFile /etc/ssl/certs/my-institution.ca-bundle 

946 

947 # ... made from additional certificates in a chain, supplied to you by 

948 # the CA. For example, mine is univcam.ca-bundle, made with the 

949 # command: 

950 # 

951 # cat TERENASSLCA.crt UTNAddTrustServer_CA.crt AddTrustExternalCARoot.crt > univcam.ca-bundle 

952 

953</VirtualHost> 

954 

955 """.strip() # noqa 

956 

957 

958# ============================================================================= 

959# Helper functions 

960# ============================================================================= 

961 

962def raise_missing(section: str, parameter: str) -> None: 

963 msg = ( 

964 f"Config file: missing/blank parameter {parameter} " 

965 f"in section [{section}]" 

966 ) 

967 raise_runtime_error(msg) 

968 

969 

970# ============================================================================= 

971# CrontabEntry 

972# ============================================================================= 

973 

974class CrontabEntry(object): 

975 """ 

976 Class to represent a ``crontab``-style entry. 

977 """ 

978 def __init__(self, 

979 line: str = None, 

980 minute: Union[str, int, List[int]] = "*", 

981 hour: Union[str, int, List[int]] = "*", 

982 day_of_week: Union[str, int, List[int]] = "*", 

983 day_of_month: Union[str, int, List[int]] = "*", 

984 month_of_year: Union[str, int, List[int]] = "*", 

985 content: str = None) -> None: 

986 """ 

987 Args: 

988 line: 

989 line of the form ``m h dow dom moy content content content``. 

990 minute: 

991 crontab "minute" entry 

992 hour: 

993 crontab "hour" entry 

994 day_of_week: 

995 crontab "day_of_week" entry 

996 day_of_month: 

997 crontab "day_of_month" entry 

998 month_of_year: 

999 crontab "month_of_year" entry 

1000 content: 

1001 crontab "thing to run" entry 

1002 

1003 If ``line`` is specified, it is used. Otherwise, the components are 

1004 used; the default for each of them is ``"*"``, meaning "all". Thus, for 

1005 example, you can specify ``minute="*/5"`` and that is sufficient to 

1006 mean "every 5 minutes". 

1007 """ 

1008 has_line = line is not None 

1009 has_components = bool(minute and hour and day_of_week and 

1010 day_of_month and month_of_year and content) 

1011 assert has_line or has_components, ( 

1012 "Specify either a crontab line or all the time components" 

1013 ) 

1014 if has_line: 

1015 line = line.split("#")[0].strip() # everything before a '#' 

1016 components = line.split() # split on whitespace 

1017 assert len(components) >= 6, ( 

1018 "Must specify 5 time components and then contents" 

1019 ) 

1020 minute, hour, day_of_week, day_of_month, month_of_year = ( 

1021 components[0:5] 

1022 ) 

1023 content = " ".join(components[5:]) 

1024 

1025 self.minute = minute 

1026 self.hour = hour 

1027 self.day_of_week = day_of_week 

1028 self.day_of_month = day_of_month 

1029 self.month_of_year = month_of_year 

1030 self.content = content 

1031 

1032 def __repr__(self) -> str: 

1033 return auto_repr(self, sort_attrs=False) 

1034 

1035 def __str__(self) -> str: 

1036 return ( 

1037 f"{self.minute} {self.hour} {self.day_of_week} " 

1038 f"{self.day_of_month} {self.month_of_year} {self.content}" 

1039 ) 

1040 

1041 def get_celery_schedule(self) -> celery.schedules.crontab: 

1042 """ 

1043 Returns the corresponding Celery schedule. 

1044 

1045 Returns: 

1046 a :class:`celery.schedules.crontab` 

1047 

1048 Raises: 

1049 :exc:`celery.schedules.ParseException` if the input can't be parsed 

1050 """ 

1051 return celery.schedules.crontab( 

1052 minute=self.minute, 

1053 hour=self.hour, 

1054 day_of_week=self.day_of_week, 

1055 day_of_month=self.day_of_month, 

1056 month_of_year=self.month_of_year, 

1057 ) 

1058 

1059 

1060# ============================================================================= 

1061# Configuration class. (It gets cached on a per-process basis.) 

1062# ============================================================================= 

1063 

1064class CamcopsConfig(object): 

1065 """ 

1066 Class representing the CamCOPS configuration. 

1067 """ 

1068 

1069 def __init__(self, 

1070 config_filename: str, 

1071 config_text: str = None) -> None: 

1072 """ 

1073 Initialize by reading the config file. 

1074 

1075 Args: 

1076 config_filename: 

1077 Filename of the config file (usual method) 

1078 config_text: 

1079 Text contents of the config file (alternative method for 

1080 special circumstances); overrides ``config_filename`` 

1081 """ 

1082 def _get_str(section: str, paramname: str, 

1083 default: str = None) -> Optional[str]: 

1084 return get_config_parameter( 

1085 parser, section, paramname, str, default) 

1086 

1087 def _get_bool(section: str, paramname: str, default: bool) -> bool: 

1088 return get_config_parameter_boolean( 

1089 parser, section, paramname, default) 

1090 

1091 def _get_int(section: str, paramname: str, 

1092 default: int = None) -> Optional[int]: 

1093 return get_config_parameter( 

1094 parser, section, paramname, int, default) 

1095 

1096 def _get_multiline(section: str, paramname: str) -> List[str]: 

1097 # http://stackoverflow.com/questions/335695/lists-in-configparser 

1098 return get_config_parameter_multiline( 

1099 parser, section, paramname, []) 

1100 

1101 def _get_multiline_ignoring_comments(section: str, 

1102 paramname: str) -> List[str]: 

1103 # Returns lines with any trailing comments removed, and any 

1104 # comment-only lines removed. 

1105 lines = _get_multiline(section, paramname) 

1106 return list(filter(None, 

1107 (x.split("#")[0].strip() for x in lines if x))) 

1108 

1109 # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ 

1110 # Learn something about our environment 

1111 # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ 

1112 self.running_under_docker = running_under_docker() 

1113 

1114 # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ 

1115 # Open config file 

1116 # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ 

1117 self.camcops_config_filename = config_filename 

1118 parser = configparser.ConfigParser() 

1119 

1120 if config_text: 

1121 log.info("Reading config from supplied string") 

1122 parser.read_string(config_text) 

1123 else: 

1124 if not config_filename: 

1125 raise AssertionError( 

1126 f"Environment variable {ENVVAR_CONFIG_FILE} not specified " 

1127 f"(and no command-line alternative given)") 

1128 log.info("Reading from config file: {!r}", config_filename) 

1129 with codecs.open(config_filename, "r", "utf8") as file: 

1130 parser.read_file(file) 

1131 

1132 # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ 

1133 # Main section (in alphabetical order) 

1134 # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ 

1135 s = CONFIG_FILE_SITE_SECTION 

1136 cs = ConfigParamSite 

1137 cd = ConfigDefaults() 

1138 

1139 self.allow_insecure_cookies = _get_bool( 

1140 s, cs.ALLOW_INSECURE_COOKIES, cd.ALLOW_INSECURE_COOKIES) 

1141 

1142 self.camcops_logo_file_absolute = _get_str( 

1143 s, cs.CAMCOPS_LOGO_FILE_ABSOLUTE, cd.CAMCOPS_LOGO_FILE_ABSOLUTE) 

1144 self.ctv_filename_spec = _get_str(s, cs.CTV_FILENAME_SPEC) 

1145 

1146 self.db_url = parser.get(s, cs.DB_URL) 

1147 # ... no default: will fail if not provided 

1148 self.db_echo = _get_bool(s, cs.DB_ECHO, cd.DB_ECHO) 

1149 self.client_api_loglevel = get_config_parameter_loglevel( 

1150 parser, s, cs.CLIENT_API_LOGLEVEL, cd.CLIENT_API_LOGLEVEL) 

1151 logging.getLogger("camcops_server.cc_modules.client_api")\ 

1152 .setLevel(self.client_api_loglevel) 

1153 # ... MUTABLE GLOBAL STATE (if relatively unimportant); todo: fix 

1154 

1155 self.disable_password_autocomplete = _get_bool( 

1156 s, cs.DISABLE_PASSWORD_AUTOCOMPLETE, 

1157 cd.DISABLE_PASSWORD_AUTOCOMPLETE) 

1158 

1159 self.email_host = _get_str(s, cs.EMAIL_HOST, "") 

1160 self.email_port = _get_int(s, cs.EMAIL_PORT, cd.EMAIL_PORT) 

1161 self.email_use_tls = _get_bool(s, cs.EMAIL_USE_TLS, cd.EMAIL_USE_TLS) 

1162 self.email_host_username = _get_str(s, cs.EMAIL_HOST_USERNAME, "") 

1163 self.email_host_password = _get_str(s, cs.EMAIL_HOST_PASSWORD, "") 

1164 

1165 self.email_from = _get_str(s, cs.EMAIL_FROM, "") 

1166 self.email_sender = _get_str(s, cs.EMAIL_SENDER, "") 

1167 self.email_reply_to = _get_str(s, cs.EMAIL_REPLY_TO, "") 

1168 

1169 self.extra_string_files = _get_multiline(s, cs.EXTRA_STRING_FILES) 

1170 

1171 self.language = _get_str(s, cs.LANGUAGE, cd.LANGUAGE) 

1172 if self.language not in POSSIBLE_LOCALES: 

1173 log.warning(f"Invalid language {self.language!r}, " 

1174 f"switching to {cd.LANGUAGE!r}") 

1175 self.language = cd.LANGUAGE 

1176 self.local_institution_url = _get_str( 

1177 s, cs.LOCAL_INSTITUTION_URL, cd.LOCAL_INSTITUTION_URL) 

1178 self.local_logo_file_absolute = _get_str( 

1179 s, cs.LOCAL_LOGO_FILE_ABSOLUTE, cd.LOCAL_LOGO_FILE_ABSOLUTE) 

1180 self.lockout_threshold = _get_int( 

1181 s, cs.LOCKOUT_THRESHOLD, cd.LOCKOUT_THRESHOLD) 

1182 self.lockout_duration_increment_minutes = _get_int( 

1183 s, cs.LOCKOUT_DURATION_INCREMENT_MINUTES, 

1184 cd.LOCKOUT_DURATION_INCREMENT_MINUTES) 

1185 

1186 self.password_change_frequency_days = _get_int( 

1187 s, cs.PASSWORD_CHANGE_FREQUENCY_DAYS, 

1188 cd.PASSWORD_CHANGE_FREQUENCY_DAYS) 

1189 self.patient_spec_if_anonymous = _get_str( 

1190 s, cs.PATIENT_SPEC_IF_ANONYMOUS, cd.PATIENT_SPEC_IF_ANONYMOUS) 

1191 self.patient_spec = _get_str(s, cs.PATIENT_SPEC) 

1192 self.permit_immediate_downloads = _get_bool( 

1193 s, cs.PERMIT_IMMEDIATE_DOWNLOADS, 

1194 cd.PERMIT_IMMEDIATE_DOWNLOADS) 

1195 # currently not configurable, but easy to add in the future: 

1196 self.plot_fontsize = cd.PLOT_FONTSIZE 

1197 

1198 self.restricted_tasks = {} # type: Dict[str, List[str]] 

1199 # ... maps XML task names to lists of authorized group names 

1200 restricted_tasks = _get_multiline(s, cs.RESTRICTED_TASKS) 

1201 for rt_line in restricted_tasks: 

1202 rt_line = rt_line.split("#")[0].strip() 

1203 # ... everything before a '#' 

1204 if not rt_line: # comment line 

1205 continue 

1206 try: 

1207 xml_taskname, groupnames = rt_line.split(":") 

1208 except ValueError: 

1209 raise ValueError( 

1210 f"Restricted tasks line not in the format " 

1211 f"'xml_taskname: groupname1, groupname2, ...'. Line was:\n" 

1212 f"{rt_line!r}" 

1213 ) 

1214 xml_taskname = xml_taskname.strip() 

1215 if xml_taskname in self.restricted_tasks: 

1216 raise ValueError(f"Duplicate restricted task specification " 

1217 f"for {xml_taskname!r}") 

1218 groupnames = [x.strip() for x in groupnames.split(",")] 

1219 for gn in groupnames: 

1220 validate_group_name(gn) 

1221 self.restricted_tasks[xml_taskname] = groupnames 

1222 

1223 self.session_timeout_minutes = _get_int( 

1224 s, cs.SESSION_TIMEOUT_MINUTES, cd.SESSION_TIMEOUT_MINUTES) 

1225 self.session_cookie_secret = _get_str(s, cs.SESSION_COOKIE_SECRET) 

1226 self.session_timeout = datetime.timedelta( 

1227 minutes=self.session_timeout_minutes) 

1228 self.snomed_task_xml_filename = _get_str( 

1229 s, cs.SNOMED_TASK_XML_FILENAME) 

1230 self.snomed_icd9_xml_filename = _get_str( 

1231 s, cs.SNOMED_ICD9_XML_FILENAME) 

1232 self.snomed_icd10_xml_filename = _get_str( 

1233 s, cs.SNOMED_ICD10_XML_FILENAME) 

1234 

1235 self.task_filename_spec = _get_str(s, cs.TASK_FILENAME_SPEC) 

1236 self.tracker_filename_spec = _get_str(s, cs.TRACKER_FILENAME_SPEC) 

1237 

1238 self.user_download_dir = _get_str(s, cs.USER_DOWNLOAD_DIR, "") 

1239 self.user_download_file_lifetime_min = _get_int( 

1240 s, cs.USER_DOWNLOAD_FILE_LIFETIME_MIN, 

1241 cd.USER_DOWNLOAD_FILE_LIFETIME_MIN) 

1242 self.user_download_max_space_mb = _get_int( 

1243 s, cs.USER_DOWNLOAD_MAX_SPACE_MB, 

1244 cd.USER_DOWNLOAD_MAX_SPACE_MB) 

1245 

1246 self.webview_loglevel = get_config_parameter_loglevel( 

1247 parser, s, cs.WEBVIEW_LOGLEVEL, cd.WEBVIEW_LOGLEVEL) 

1248 logging.getLogger().setLevel(self.webview_loglevel) # root logger 

1249 # ... MUTABLE GLOBAL STATE (if relatively unimportant); todo: fix 

1250 self.wkhtmltopdf_filename = _get_str(s, cs.WKHTMLTOPDF_FILENAME) 

1251 

1252 # More validity checks for the main section: 

1253 if not self.patient_spec_if_anonymous: 

1254 raise_missing(s, cs.PATIENT_SPEC_IF_ANONYMOUS) 

1255 if not self.patient_spec: 

1256 raise_missing(s, cs.PATIENT_SPEC) 

1257 if not self.session_cookie_secret: 

1258 raise_missing(s, cs.SESSION_COOKIE_SECRET) 

1259 if not self.task_filename_spec: 

1260 raise_missing(s, cs.TASK_FILENAME_SPEC) 

1261 if not self.tracker_filename_spec: 

1262 raise_missing(s, cs.TRACKER_FILENAME_SPEC) 

1263 if not self.ctv_filename_spec: 

1264 raise_missing(s, cs.CTV_FILENAME_SPEC) 

1265 

1266 # To prevent errors: 

1267 del s 

1268 del cs 

1269 

1270 # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ 

1271 # Web server/WSGI section 

1272 # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ 

1273 ws = CONFIG_FILE_SERVER_SECTION 

1274 cw = ConfigParamServer 

1275 

1276 self.cherrypy_log_screen = _get_bool(ws, cw.CHERRYPY_LOG_SCREEN, 

1277 cd.CHERRYPY_LOG_SCREEN) 

1278 self.cherrypy_root_path = _get_str( 

1279 ws, cw.CHERRYPY_ROOT_PATH, cd.CHERRYPY_ROOT_PATH) 

1280 self.cherrypy_server_name = _get_str( 

1281 ws, cw.CHERRYPY_SERVER_NAME, cd.CHERRYPY_SERVER_NAME) 

1282 self.cherrypy_threads_max = _get_int( 

1283 ws, cw.CHERRYPY_THREADS_MAX, cd.CHERRYPY_THREADS_MAX) 

1284 self.cherrypy_threads_start = _get_int( 

1285 ws, cw.CHERRYPY_THREADS_START, cd.CHERRYPY_THREADS_START) 

1286 self.debug_reverse_proxy = _get_bool(ws, cw.DEBUG_REVERSE_PROXY, 

1287 cd.DEBUG_REVERSE_PROXY) 

1288 self.debug_show_gunicorn_options = _get_bool( 

1289 ws, cw.DEBUG_SHOW_GUNICORN_OPTIONS, cd.DEBUG_SHOW_GUNICORN_OPTIONS) 

1290 self.debug_toolbar = _get_bool(ws, cw.DEBUG_TOOLBAR, cd.DEBUG_TOOLBAR) 

1291 self.gunicorn_debug_reload = _get_bool( 

1292 ws, cw.GUNICORN_DEBUG_RELOAD, cd.GUNICORN_DEBUG_RELOAD) 

1293 self.gunicorn_num_workers = _get_int( 

1294 ws, cw.GUNICORN_NUM_WORKERS, cd.GUNICORN_NUM_WORKERS) 

1295 self.gunicorn_timeout_s = _get_int( 

1296 ws, cw.GUNICORN_TIMEOUT_S, cd.GUNICORN_TIMEOUT_S) 

1297 self.host = _get_str(ws, cw.HOST, cd.HOST) 

1298 self.port = _get_int(ws, cw.PORT, cd.PORT) 

1299 self.proxy_http_host = _get_str(ws, cw.PROXY_HTTP_HOST) 

1300 self.proxy_remote_addr = _get_str(ws, cw.PROXY_REMOTE_ADDR) 

1301 self.proxy_rewrite_path_info = _get_bool( 

1302 ws, cw.PROXY_REWRITE_PATH_INFO, cd.PROXY_REWRITE_PATH_INFO) 

1303 self.proxy_script_name = _get_str(ws, cw.PROXY_SCRIPT_NAME) 

1304 self.proxy_server_name = _get_str(ws, cw.PROXY_SERVER_NAME) 

1305 self.proxy_server_port = _get_int(ws, cw.PROXY_SERVER_PORT) 

1306 self.proxy_url_scheme = _get_str(ws, cw.PROXY_URL_SCHEME) 

1307 self.show_request_immediately = _get_bool( 

1308 ws, cw.SHOW_REQUEST_IMMEDIATELY, cd.SHOW_REQUEST_IMMEDIATELY) 

1309 self.show_requests = _get_bool(ws, cw.SHOW_REQUESTS, cd.SHOW_REQUESTS) 

1310 self.show_response = _get_bool(ws, cw.SHOW_RESPONSE, cd.SHOW_RESPONSE) 

1311 self.show_timing = _get_bool(ws, cw.SHOW_TIMING, cd.SHOW_TIMING) 

1312 self.ssl_certificate = _get_str(ws, cw.SSL_CERTIFICATE) 

1313 self.ssl_private_key = _get_str(ws, cw.SSL_PRIVATE_KEY) 

1314 self.static_cache_duration_s = _get_int(ws, cw.STATIC_CACHE_DURATION_S, 

1315 cd.STATIC_CACHE_DURATION_S) 

1316 self.trusted_proxy_headers = _get_multiline( 

1317 ws, cw.TRUSTED_PROXY_HEADERS) 

1318 self.unix_domain_socket = _get_str(ws, cw.UNIX_DOMAIN_SOCKET) 

1319 

1320 for tph in self.trusted_proxy_headers: 

1321 if tph not in ReverseProxiedMiddleware.ALL_CANDIDATES: 

1322 raise ValueError( 

1323 f"Invalid {cw.TRUSTED_PROXY_HEADERS} value specified: " 

1324 f"was {tph!r}, options are " 

1325 f"{ReverseProxiedMiddleware.ALL_CANDIDATES}") 

1326 

1327 del ws 

1328 del cw 

1329 

1330 # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ 

1331 # Export section 

1332 # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ 

1333 es = CONFIG_FILE_EXPORT_SECTION 

1334 ce = ConfigParamExportGeneral 

1335 

1336 self.celery_beat_extra_args = _get_multiline( 

1337 es, ce.CELERY_BEAT_EXTRA_ARGS) 

1338 self.celery_beat_schedule_database = _get_str( 

1339 es, ce.CELERY_BEAT_SCHEDULE_DATABASE) 

1340 if not self.celery_beat_schedule_database: 

1341 raise_missing(es, ce.CELERY_BEAT_SCHEDULE_DATABASE) 

1342 self.celery_broker_url = _get_str( 

1343 es, ce.CELERY_BROKER_URL, cd.CELERY_BROKER_URL) 

1344 self.celery_worker_extra_args = _get_multiline( 

1345 es, ce.CELERY_WORKER_EXTRA_ARGS) 

1346 self.celery_export_task_rate_limit = _get_str( 

1347 es, ce.CELERY_EXPORT_TASK_RATE_LIMIT) 

1348 

1349 self.export_lockdir = _get_str(es, ce.EXPORT_LOCKDIR) 

1350 if not self.export_lockdir: 

1351 raise_missing(es, ConfigParamExportGeneral.EXPORT_LOCKDIR) 

1352 

1353 self.export_recipient_names = _get_multiline_ignoring_comments( 

1354 CONFIG_FILE_EXPORT_SECTION, ce.RECIPIENTS) 

1355 duplicates = [name for name, count in 

1356 collections.Counter(self.export_recipient_names).items() 

1357 if count > 1] 

1358 if duplicates: 

1359 raise ValueError( 

1360 f"Duplicate export recipients specified: {duplicates!r}") 

1361 for recip_name in self.export_recipient_names: 

1362 if re.match(VALID_RECIPIENT_NAME_REGEX, recip_name) is None: 

1363 raise ValueError( 

1364 f"Recipient names must be alphanumeric or _- only; was " 

1365 f"{recip_name!r}") 

1366 if len(set(self.export_recipient_names)) != len(self.export_recipient_names): # noqa 

1367 raise ValueError("Recipient names contain duplicates") 

1368 self._export_recipients = [] # type: List[ExportRecipientInfo] 

1369 self._read_export_recipients(parser) 

1370 

1371 self.schedule_timezone = _get_str( 

1372 es, ce.SCHEDULE_TIMEZONE, cd.SCHEDULE_TIMEZONE) 

1373 

1374 self.crontab_entries = [] # type: List[CrontabEntry] 

1375 crontab_lines = _get_multiline(es, ce.SCHEDULE) 

1376 for crontab_line in crontab_lines: 

1377 crontab_line = crontab_line.split("#")[0].strip() 

1378 # ... everything before a '#' 

1379 if not crontab_line: # comment line 

1380 continue 

1381 crontab_entry = CrontabEntry(line=crontab_line) 

1382 if crontab_entry.content not in self.export_recipient_names: 

1383 raise ValueError( 

1384 f"{ce.SCHEDULE} setting exists for non-existent recipient " 

1385 f"{crontab_entry.content}") 

1386 self.crontab_entries.append(crontab_entry) 

1387 

1388 del es 

1389 del ce 

1390 

1391 # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ 

1392 # Other attributes 

1393 # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ 

1394 self._sqla_engine = None 

1395 

1396 # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ 

1397 # Docker checks 

1398 # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ 

1399 if self.running_under_docker: 

1400 log.info("Docker environment detected") 

1401 

1402 # Values expected to be fixed 

1403 warn_if_not_docker_value( 

1404 param_name=ConfigParamExportGeneral.CELERY_BROKER_URL, 

1405 actual_value=self.celery_broker_url, 

1406 required_value=DockerConstants.CELERY_BROKER_URL 

1407 ) 

1408 warn_if_not_docker_value( 

1409 param_name=ConfigParamServer.HOST, 

1410 actual_value=self.host, 

1411 required_value=DockerConstants.HOST 

1412 ) 

1413 

1414 # Values expected to be present 

1415 # 

1416 # - Re SSL certificates: reconsidered. People may want to run 

1417 # internal plain HTTP but then an Apache front end, and they 

1418 # wouldn't appreciate the warnings. 

1419 # 

1420 # warn_if_not_present( 

1421 # param_name=ConfigParamServer.SSL_CERTIFICATE, 

1422 # value=self.ssl_certificate 

1423 # ) 

1424 # warn_if_not_present( 

1425 # param_name=ConfigParamServer.SSL_PRIVATE_KEY, 

1426 # value=self.ssl_private_key 

1427 # ) 

1428 

1429 # Config-related files 

1430 warn_if_not_within_docker_dir( 

1431 param_name=ConfigParamServer.SSL_CERTIFICATE, 

1432 filespec=self.ssl_certificate, 

1433 permit_cfg=True 

1434 ) 

1435 warn_if_not_within_docker_dir( 

1436 param_name=ConfigParamServer.SSL_PRIVATE_KEY, 

1437 filespec=self.ssl_private_key, 

1438 permit_cfg=True 

1439 ) 

1440 warn_if_not_within_docker_dir( 

1441 param_name=ConfigParamSite.LOCAL_LOGO_FILE_ABSOLUTE, 

1442 filespec=self.local_logo_file_absolute, 

1443 permit_cfg=True, 

1444 permit_venv=True 

1445 ) 

1446 warn_if_not_within_docker_dir( 

1447 param_name=ConfigParamSite.CAMCOPS_LOGO_FILE_ABSOLUTE, 

1448 filespec=self.camcops_logo_file_absolute, 

1449 permit_cfg=True, 

1450 permit_venv=True 

1451 ) 

1452 for esf in self.extra_string_files: 

1453 warn_if_not_within_docker_dir( 

1454 param_name=ConfigParamSite.EXTRA_STRING_FILES, 

1455 filespec=esf, 

1456 permit_cfg=True, 

1457 permit_venv=True, 

1458 param_contains_not_is=True 

1459 ) 

1460 warn_if_not_within_docker_dir( 

1461 param_name=ConfigParamSite.SNOMED_ICD9_XML_FILENAME, 

1462 filespec=self.snomed_icd9_xml_filename, 

1463 permit_cfg=True, 

1464 permit_venv=True 

1465 ) 

1466 warn_if_not_within_docker_dir( 

1467 param_name=ConfigParamSite.SNOMED_ICD10_XML_FILENAME, 

1468 filespec=self.snomed_icd10_xml_filename, 

1469 permit_cfg=True, 

1470 permit_venv=True 

1471 ) 

1472 warn_if_not_within_docker_dir( 

1473 param_name=ConfigParamSite.SNOMED_TASK_XML_FILENAME, 

1474 filespec=self.snomed_task_xml_filename, 

1475 permit_cfg=True, 

1476 permit_venv=True 

1477 ) 

1478 

1479 # Temporary/scratch space that needs to be shared between Docker 

1480 # containers 

1481 warn_if_not_within_docker_dir( 

1482 param_name=ConfigParamSite.USER_DOWNLOAD_DIR, 

1483 filespec=self.user_download_dir, 

1484 permit_tmp=True 

1485 ) 

1486 warn_if_not_within_docker_dir( 

1487 param_name=ConfigParamExportGeneral.CELERY_BEAT_SCHEDULE_DATABASE, # noqa 

1488 filespec=self.celery_beat_schedule_database, 

1489 permit_tmp=True 

1490 ) 

1491 warn_if_not_within_docker_dir( 

1492 param_name=ConfigParamExportGeneral.EXPORT_LOCKDIR, 

1493 filespec=self.export_lockdir, 

1494 permit_tmp=True 

1495 ) 

1496 

1497 # ------------------------------------------------------------------------- 

1498 # Database functions 

1499 # ------------------------------------------------------------------------- 

1500 

1501 def get_sqla_engine(self) -> Engine: 

1502 """ 

1503 Returns an SQLAlchemy :class:`Engine`. 

1504 

1505 I was previously misinterpreting the appropriate scope of an Engine. 

1506 I thought: create one per request. 

1507 But the Engine represents the connection *pool*. 

1508 So if you create them all the time, you get e.g. a 

1509 'Too many connections' error. 

1510 

1511 "The appropriate scope is once per [database] URL per application, 

1512 at the module level." 

1513 

1514 - https://groups.google.com/forum/#!topic/sqlalchemy/ZtCo2DsHhS4 

1515 - https://stackoverflow.com/questions/8645250/how-to-close-sqlalchemy-connection-in-mysql 

1516 

1517 Now, our CamcopsConfig instance is cached, so there should be one of 

1518 them overall. See get_config() below. 

1519 

1520 Therefore, making the engine a member of this class should do the 

1521 trick, whilst avoiding global variables. 

1522 """ # noqa 

1523 if self._sqla_engine is None: 

1524 self._sqla_engine = create_engine( 

1525 self.db_url, 

1526 echo=self.db_echo, 

1527 pool_pre_ping=True, 

1528 # pool_size=0, # no limit (for parallel testing, which failed) 

1529 ) 

1530 log.debug("Created SQLAlchemy engine for URL {}", 

1531 get_safe_url_from_engine(self._sqla_engine)) 

1532 return self._sqla_engine 

1533 

1534 @property 

1535 @cache_region_static.cache_on_arguments(function_key_generator=fkg) 

1536 def get_all_table_names(self) -> List[str]: 

1537 """ 

1538 Returns all table names from the database. 

1539 """ 

1540 log.debug("Fetching database table names") 

1541 engine = self.get_sqla_engine() 

1542 return get_table_names(engine=engine) 

1543 

1544 def get_dbsession_raw(self) -> SqlASession: 

1545 """ 

1546 Returns a raw SQLAlchemy Session. 

1547 Avoid this -- use :func:`get_dbsession_context` instead. 

1548 """ 

1549 engine = self.get_sqla_engine() 

1550 maker = sessionmaker(bind=engine) 

1551 dbsession = maker() # type: SqlASession 

1552 return dbsession 

1553 

1554 @contextlib.contextmanager 

1555 def get_dbsession_context(self) -> Generator[SqlASession, None, None]: 

1556 """ 

1557 Context manager to provide an SQLAlchemy session that will COMMIT 

1558 once we've finished, or perform a ROLLBACK if there was an exception. 

1559 """ 

1560 dbsession = self.get_dbsession_raw() 

1561 # noinspection PyBroadException 

1562 try: 

1563 yield dbsession 

1564 dbsession.commit() 

1565 except Exception: 

1566 dbsession.rollback() 

1567 raise 

1568 finally: 

1569 dbsession.close() 

1570 

1571 def _assert_valid_database_engine(self) -> None: 

1572 """ 

1573 Assert that our backend database is a valid type. 

1574 

1575 Specifically, we prohibit: 

1576 

1577 - SQL Server versions before 2008: they don't support timezones 

1578 and we need that. 

1579 """ 

1580 engine = self.get_sqla_engine() 

1581 if not is_sqlserver(engine): 

1582 return 

1583 assert is_sqlserver_2008_or_later(engine), ( 

1584 "If you use Microsoft SQL Server as the back-end database for a " 

1585 "CamCOPS server, it must be at least SQL Server 2008. Older " 

1586 "versions do not have time zone awareness." 

1587 ) 

1588 

1589 def _assert_database_is_at_head(self) -> None: 

1590 """ 

1591 Assert that the current database is at its head (most recent) revision, 

1592 by comparing its Alembic version number (written into the Alembic 

1593 version table of the database) to the most recent Alembic revision in 

1594 our ``camcops_server/alembic/versions`` directory. 

1595 """ 

1596 current, head = get_current_and_head_revision( 

1597 database_url=self.db_url, 

1598 alembic_config_filename=ALEMBIC_CONFIG_FILENAME, 

1599 alembic_base_dir=ALEMBIC_BASE_DIR, 

1600 version_table=ALEMBIC_VERSION_TABLE, 

1601 ) 

1602 if current == head: 

1603 log.debug("Database is at correct (head) revision of {}", current) 

1604 else: 

1605 raise_runtime_error( 

1606 f"Database structure is at version {current} but should be at " 

1607 f"version {head}. CamCOPS will not start. Please use the " 

1608 f"'upgrade_db' command to fix this.") 

1609 

1610 def assert_database_ok(self) -> None: 

1611 """ 

1612 Asserts that our database engine is OK and our database structure is 

1613 correct. 

1614 """ 

1615 self._assert_valid_database_engine() 

1616 self._assert_database_is_at_head() 

1617 

1618 # ------------------------------------------------------------------------- 

1619 # SNOMED-CT functions 

1620 # ------------------------------------------------------------------------- 

1621 

1622 def get_task_snomed_concepts(self) -> Dict[str, SnomedConcept]: 

1623 """ 

1624 Returns all SNOMED-CT concepts for tasks. 

1625 

1626 Returns: 

1627 dict: maps lookup strings to :class:`SnomedConcept` objects 

1628 """ 

1629 if not self.snomed_task_xml_filename: 

1630 return {} 

1631 return get_all_task_snomed_concepts(self.snomed_task_xml_filename) 

1632 

1633 def get_icd9cm_snomed_concepts(self) -> Dict[str, List[SnomedConcept]]: 

1634 """ 

1635 Returns all SNOMED-CT concepts for ICD-9-CM codes supported by CamCOPS. 

1636 

1637 Returns: 

1638 dict: maps ICD-9-CM codes to :class:`SnomedConcept` objects 

1639 """ 

1640 if not self.snomed_icd9_xml_filename: 

1641 return {} 

1642 return get_icd9_snomed_concepts_from_xml(self.snomed_icd9_xml_filename) 

1643 

1644 def get_icd10_snomed_concepts(self) -> Dict[str, List[SnomedConcept]]: 

1645 """ 

1646 Returns all SNOMED-CT concepts for ICD-10-CM codes supported by 

1647 CamCOPS. 

1648 

1649 Returns: 

1650 dict: maps ICD-10 codes to :class:`SnomedConcept` objects 

1651 """ 

1652 if not self.snomed_icd10_xml_filename: 

1653 return {} 

1654 return get_icd10_snomed_concepts_from_xml( 

1655 self.snomed_icd10_xml_filename) 

1656 

1657 # ------------------------------------------------------------------------- 

1658 # Export functions 

1659 # ------------------------------------------------------------------------- 

1660 

1661 def _read_export_recipients( 

1662 self, 

1663 parser: configparser.ConfigParser = None) -> None: 

1664 """ 

1665 Loads 

1666 :class:`camcops_server.cc_modules.cc_exportrecipientinfo.ExportRecipientInfo` 

1667 objects from the config file. Stores them in 

1668 ``self._export_recipients``. 

1669 

1670 Note that these objects are **not** associated with a database session. 

1671 

1672 Args: 

1673 parser: optional :class:`configparser.ConfigParser` object. 

1674 """ 

1675 self._export_recipients = [] # type: List[ExportRecipientInfo] 

1676 for recip_name in self.export_recipient_names: 

1677 log.debug("Loading export config for recipient {!r}", recip_name) 

1678 try: 

1679 validate_export_recipient_name(recip_name) 

1680 except ValueError as e: 

1681 raise ValueError(f"Bad recipient name {recip_name!r}: {e}") 

1682 recipient = ExportRecipientInfo.read_from_config( 

1683 self, parser=parser, recipient_name=recip_name) 

1684 self._export_recipients.append(recipient) 

1685 

1686 def get_all_export_recipient_info(self) -> List["ExportRecipientInfo"]: 

1687 """ 

1688 Returns all export recipients (in their "database unaware" form) 

1689 specified in the config. 

1690 

1691 Returns: 

1692 list: of 

1693 :class:`camcops_server.cc_modules.cc_exportrecipientinfo.ExportRecipientInfo` 

1694 """ # noqa 

1695 return self._export_recipients 

1696 

1697 # ------------------------------------------------------------------------- 

1698 # File-based locks 

1699 # ------------------------------------------------------------------------- 

1700 

1701 def get_export_lockfilename_db(self, recipient_name: str) -> str: 

1702 """ 

1703 Returns a full path to a lockfile suitable for locking for a 

1704 whole-database export to a particular export recipient. 

1705 

1706 Args: 

1707 recipient_name: name of the recipient 

1708 

1709 Returns: 

1710 a filename 

1711 """ 

1712 filename = f"camcops_export_db_{recipient_name}" 

1713 # ".lock" is appended automatically by the lockfile package 

1714 return os.path.join(self.export_lockdir, filename) 

1715 

1716 def get_export_lockfilename_task(self, recipient_name: str, 

1717 basetable: str, pk: int) -> str: 

1718 """ 

1719 Returns a full path to a lockfile suitable for locking for a 

1720 single-task export to a particular export recipient. 

1721 

1722 Args: 

1723 recipient_name: name of the recipient 

1724 basetable: task base table name 

1725 pk: server PK of the task 

1726 

1727 Returns: 

1728 a filename 

1729 """ 

1730 filename = f"camcops_export_task_{recipient_name}_{basetable}_{pk}" 

1731 # ".lock" is appended automatically by the lockfile package 

1732 return os.path.join(self.export_lockdir, filename) 

1733 

1734 def get_master_export_recipient_lockfilename(self) -> str: 

1735 """ 

1736 When we are modifying export recipients, we check "is this information 

1737 the same as the current version in the database", and if not, we write 

1738 fresh information to the database. If lots of processes do that at the 

1739 same time, we have a problem (usually a database deadlock) -- hence 

1740 this lock. 

1741 

1742 Returns: 

1743 a filename 

1744 """ 

1745 filename = "camcops_master_export_recipient" 

1746 # ".lock" is appended automatically by the lockfile package 

1747 return os.path.join(self.export_lockdir, filename) 

1748 

1749 def get_celery_beat_pidfilename(self) -> str: 

1750 """ 

1751 Process ID file (pidfile) used by ``celery beat --pidfile ...``. 

1752 """ 

1753 filename = "camcops_celerybeat.pid" 

1754 return os.path.join(self.export_lockdir, filename) 

1755 

1756 

1757# ============================================================================= 

1758# Get config filename from an appropriate environment (WSGI or OS) 

1759# ============================================================================= 

1760 

1761def get_config_filename_from_os_env() -> str: 

1762 """ 

1763 Returns the config filename to use, from our operating system environment 

1764 variable. 

1765 

1766 (We do NOT trust the WSGI environment for this.) 

1767 """ 

1768 config_filename = os.environ.get(ENVVAR_CONFIG_FILE) 

1769 if not config_filename: 

1770 raise AssertionError( 

1771 f"OS environment did not provide the required " 

1772 f"environment variable {ENVVAR_CONFIG_FILE}") 

1773 return config_filename 

1774 

1775 

1776# ============================================================================= 

1777# Cached instances 

1778# ============================================================================= 

1779 

1780@cache_region_static.cache_on_arguments(function_key_generator=fkg) 

1781def get_config(config_filename: str) -> CamcopsConfig: 

1782 """ 

1783 Returns a :class:`camcops_server.cc_modules.cc_config.CamcopsConfig` from 

1784 the specified config filename. 

1785 

1786 Cached. 

1787 """ 

1788 return CamcopsConfig(config_filename) 

1789 

1790 

1791# ============================================================================= 

1792# Get default config 

1793# ============================================================================= 

1794 

1795def get_default_config_from_os_env() -> CamcopsConfig: 

1796 """ 

1797 Returns the :class:`camcops_server.cc_modules.cc_config.CamcopsConfig` 

1798 representing the config filename that we read from our operating system 

1799 environment variable. 

1800 """ 

1801 if ON_READTHEDOCS: 

1802 return CamcopsConfig(config_filename="", config_text=get_demo_config()) 

1803 else: 

1804 return get_config(get_config_filename_from_os_env())