SHELL=/usr/bin/bash
BASH_ENV="/local/home/lstanalyzer/.bashrc"
CRON_ENV="/local/home/lstanalyzer/osa-env.sh"
CRON_TZ=UTC
LAUNCHERS_DIR="/fefs/aswg/workspace/maria.rivero/lstosa/src/osa/crontab/launchers"
# Update astropy auxiliary data (e.g. IERS) every week
0 0 * * SUN source /fefs/aswg/lstosa/utils/osa-conda; python /fefs/aswg/data/aux/astropy/get_iers_data.py  > /dev/null 2>&1
#
# lstosa jobs
# -------------------------------------------------------------------------------------------------------------------------
# [COPY OBS OVERVIEW] Copy observations overview and daily camera checks to lst1 datacheck webserver
# -------------------------------------------------------------------------------------------------------------------------
00 09 * * * /usr/bin/bash /fefs/aswg/lstosa/misc/copy_camera_daily_check_to_www.sh >/dev/null 2>&1
#
# -------------------------------------------------------------------------------------------------------------------------
# [RUN SUMMARY] Produce the run summary file at the end of the observations to make sure it contains all the runs taken
# -------------------------------------------------------------------------------------------------------------------------
55 06 * * * obsdate=`date +\%Y\%m\%d -d yesterday`; export obsdate; source /fefs/aswg/lstosa/utils/osa-conda; lstchain_create_run_summary -d $obsdate -o /fefs/onsite/data/lst-pipe/LSTN-01/monitoring/RunSummary --r0-path=/fefs/onsite/data/R0/LSTN-01/lst-arraydaq/events --overwrite > /dev/null 2>&1
#
# -------------------------------------------------------------------------------------------------------------------------
# [MERGED RUN SUMMARY] Produce the merged run summary file containing all the runs taken by the LST.
#                      This only appends the runs of the given date.
# -------------------------------------------------------------------------------------------------------------------------
00 07 * * * obsdate=`date +\%Y\%m\%d -d yesterday`; source /fefs/aswg/lstosa/utils/osa-conda; lstchain_merge_run_summaries -d $obsdate /fefs/onsite/data/lst-pipe/LSTN-01/OSA/Catalog/merged_RunSummary.ecsv --monitoring-path=/fefs/onsite/data/lst-pipe/LSTN-01/monitoring > /dev/null 2>&1
#
# -------------------------------------------------------------------------------------------------------------------------
# [GAIN SELECTION] Launch Gain Selection process 
# -------------------------------------------------------------------------------------------------------------------------
32 07-18 * * * $LAUNCHERS_DIR/launch_GainSel.sh
#
# -------------------------------------------------------------------------------------------------------------------------
# [GAIN SELECTION CHECK & GAIN SEL WEB MAKER] Make gain selection check and xhtml table and copy it to the lst1 webserver
# -------------------------------------------------------------------------------------------------------------------------
00,15,30,45 08-19 * * * $LAUNCHERS_DIR/launch_GainSelCheck.sh
#
# -------------------------------------------------------------------------------------------------------------------------
# [SIMULATED SEQUENCER] Launch sequencer once in the morning simulated. It also produces the run summary file if not found.
# -------------------------------------------------------------------------------------------------------------------------
05 08 * * * . "$CRON_ENV"; source "$CONDA_ENV"; sequencer -c "$CFG" -s -d "$OBS_DATE" LST1 > /dev/null 2>&1
#
#  -------------------------------------------------------------------------------------------------------------------------
# [SEQUENCER 1] Run only up to DL1A
# -------------------------------------------------------------------------------------------------------------------------
20,50 07-19 * * * $LAUNCHERS_DIR/launch_Sequencer1.sh
#
#  -------------------------------------------------------------------------------------------------------------------------
# [SEQUENCER_CATB_TAILCUTS] Runs from DL1A, produces Cat-B calibration and tailcut files
# -------------------------------------------------------------------------------------------------------------------------
10,50 07-20 * * * $LAUNCHERS_DIR/launch_SequencerCatB.sh
#
# -------------------------------------------------------------------------------------------------------------------------
# [SEQUENCER 2] Finishes the processing producing DL1AB and DL2
# -------------------------------------------------------------------------------------------------------------------------
59,30 08-20 * * * $LAUNCHERS_DIR/launch_Sequencer2.sh
#
# -------------------------------------------------------------------------------------------------------------------------
# [SEQUENCER WEB] Make sequencer xhtml table and copy it to the lst1 webserver
# -------------------------------------------------------------------------------------------------------------------------
*/20 07-22 * * * $LAUNCHERS_DIR/launch_SequencerWeb.sh
#
# -------------------------------------------------------------------------------------------------------------------------
# [AUTOCLOSER] Launch the closer without forcing it (no -f option).
# -------------------------------------------------------------------------------------------------------------------------
25,55 9-23 * * * $LAUNCHERS_DIR/launch_Autocloser.sh
#
# -------------------------------------------------------------------------------------------------------------------------
# [COPY DATACHECK] Copy the available calibration and DL1 datacheck to the LST1 webserver.
# -------------------------------------------------------------------------------------------------------------------------
05,25,45 07-23 * * * $LAUNCHERS_DIR/launch_Datacheck.sh
#
# -------------------------------------------------------------------------------------------------------------------------
# [SOURCE CATALOG]
# -------------------------------------------------------------------------------------------------------------------------
52 10-23 * * * obsdate=`date +\%Y-\%m-\%d -d yesterday`; export obsdate; source /fefs/aswg/lstosa/utils/osa-conda; update_source_catalog -c /fefs/aswg/lstosa/cfg/sequencer_v0.11.cfg -d $obsdate > /fefs/onsite/data/lst-pipe/LSTN-01/monitoring/RunCatalog/log/Source-catalog-$obsdate 2>&1
#
#  ------------------------------------------------------------------------------------------------------------------------
# [COPY CUMULATIVE LST-1 OBSERVATION TIME]
# (added by Daniel Morcuende on 2024-04-05)
# -------------------------------------------------------------------------------------------------------------------------
0 18 *  * FRI /usr/bin/bash /fefs/aswg/lstosa/misc/update_cumulative_obs_time.sh >/dev/null 2>&1
#
# -------------------------------------------------------------------------------------------------------------------------
#TROUBLESHOOTING
# -------------------------------------------------------------------------------------------------------------------------
#*/20 7-22 * * * obsdate=`date +\%Y-\%m-\%d`; cd /fefs/aswg/lstosa/troubleshooting/ && /fefs/aswg/software/conda/bin/python troubleshooting.py >> /fefs/aswg/lstosa/troubleshooting/log/cron_log_${obsdate}.log 2>&1
#
# -------------------------------------------------------------------------------------------------------------------------
# Daniel Morcuende (2023-11-08) periodically remove leftover jobs kept in the SLURM as pending after job dependency not 
# satisfied because of failure (Reason in squeue: 'DependencyNeverSatisfied').
# It does only look for lstosa jobs launched by sequencer with the prefix "LST1_" and also the merging jobs launched by 
# autocloser like "longterm_dl1_datacheck".
# -------------------------------------------------------------------------------------------------------------------------
*/30 * * * * /local/home/lstanalyzer/remove_dependency_never_satisfied_jobs.sh > /dev/null 2>&1
#
# -------------------------------------------------------------------------------------------------------------------------
# Restart camera processing of calibration runs in realtime in case it was killed for whatever reason
# -------------------------------------------------------------------------------------------------------------------------
04 * * * * cd /local/home/lstanalyzer/camera/lst-camera-watchers;./lst1_watcher_restarter
