From fd95baa09a45fa657cd83144d63e0110f8d75a59 Mon Sep 17 00:00:00 2001 From: Brian Woods Date: Sun, 19 Mar 2023 15:34:13 -0400 Subject: queue: add base files for queuing system Add a simple command queuing system. Simple but useful for adding multiple commands to a queue to run sequentially. --- queue/README.md | 24 +++++++++++ queue/queue_add.sh | 60 +++++++++++++++++++++++++++ queue/queue_worker.sh | 113 ++++++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 197 insertions(+) create mode 100644 queue/README.md create mode 100755 queue/queue_add.sh create mode 100755 queue/queue_worker.sh (limited to 'queue') diff --git a/queue/README.md b/queue/README.md new file mode 100644 index 0000000..be3fbdd --- /dev/null +++ b/queue/README.md @@ -0,0 +1,24 @@ +# Queue scripts + +Basically these scripts are used for throwing commands in a queue and +then running them sequentially with a configurable about of time between +them. +This allows for just throwing things in the queue and forgetting about +them without having to manually run commands one after the other. + +## Usage +Configuring the paths is highly suggested. You can have everything +other than the queue\_add.sh script in a queue directory. +There's also the `CMD` variable useful if you're just running one +command with differing or even the same options. +The `TIME_BASE` and `TIME_RND` are useful to spacing the queue tasks +out if need be. + +## Example + ./queue_add.sh 'echo 1 >> log.txt' + ./queue_add.sh 'echo 2 >> log.txt' + ./queue_add.sh 'echo 3 >> log.txt' + ./queue_add.sh 'echo 4 >> log.txt' + ./queue_add.sh 'echo 5 >> log.txt' + sleep 1 # give it enough time to run the first job + tail -f log.txt # see the output diff --git a/queue/queue_add.sh b/queue/queue_add.sh new file mode 100755 index 0000000..74fb2df --- /dev/null +++ b/queue/queue_add.sh @@ -0,0 +1,60 @@ +#!/bin/bash +# SPDX-FileCopyrightText: 2023 Brian Woods +# SPDX-License-Identifier: GPL-2.0-or-later + +# paths +LOCK="queue.lock" +QUEUE="queue.file" +WORKER="./queue_worker.sh" +WORKER_PID="queue_worker.pid" +WORKER_LOG="queue_worker.log" + +queue_push () +{ + local lock_fd="3" + # aquire lock + if ! exec {lock_fd}>"$LOCK" ; then + echo "Can't get lockfile fd" + return + fi + + if ! flock -w 10 "$lock_fd" ; then + echo "couldn't get lock" + return + fi + + # actually write it out + echo "$@" >> "$QUEUE" + + # release lock + flock -u "$lock_fd" + exec {lock_fd}>&- +} + +spawn_worker () +{ + nohup "$WORKER" >> "$WORKER_LOG" & + + # 5 seconds of waiting + local max=50 + local tries=0 + while [ "$tries" -lt "$max" ] ; do + sleep .1 + tries=$(( tries + 1 )) + + if [ -f "$WORKER_PID" ] ; then + break + fi + done + if [ "$tries" = "$max" ] ; then + echo "warning waiting period without worker PID created" + fi +} + +queue_push $@ + +# need to do some locking if we want this to be actually safe +# 1. get lock, 2. spin off worker, 3. wait until pid, 4. release lock +if [ ! -f "$WORKER_PID" ] ; then + spawn_worker +fi diff --git a/queue/queue_worker.sh b/queue/queue_worker.sh new file mode 100755 index 0000000..8b2b0b2 --- /dev/null +++ b/queue/queue_worker.sh @@ -0,0 +1,113 @@ +#!/bin/bash +# SPDX-FileCopyrightText: 2023 Brian Woods +# SPDX-License-Identifier: GPL-2.0-or-later + +# settings +CMD="" +TIME_BASE=10 # always wait this time +TIME_RND=0 # above plus 0-120 + +# pathes +LOCK="queue.lock" +QUEUE="queue.file" +WORKER_PID="queue_worker.pid" + +# references for why I went with sed, that and ease use +# https://www.baeldung.com/linux/remove-first-line-text-file + +queue_shift () +{ + queue_item="" + local lock_fd="3" + # aquire lock + if ! exec {lock_fd}>"$LOCK" ; then + echo "Can't get lockfile fd" + return + fi + + if ! flock -w 10 "$lock_fd" ; then + echo "couldn't get lock" + return + fi + + if [ -f "$QUEUE" ] ; then + local line="$(head -n 1 $QUEUE)" + sed -i '1d' "$QUEUE" + + #delete if empty + if [ ! -s "$QUEUE" ] ; then + rm "$QUEUE" + fi + fi + + # release lock + flock -u "$lock_fd" + exec {lock_fd}>&- + + queue_item="$line" +} + +queue_work () +{ + local rv=1 + local lock_fd="3" + # aquire lock + if ! exec {lock_fd}>"$LOCK" ; then + echo "Can't get lockfile fd" + return + fi + + if ! flock -w 10 "$lock_fd" ; then + echo "couldn't get lock" + return + fi + + # we have work + if [ -f "$QUEUE" ] ; then + rv=0 + fi + + # release lock + flock -u "$lock_fd" + exec {lock_fd}>&- + + return $rv +} + +sleep_pause () +{ + if [ "$TIME_RND" = "0" ] ; then + local sleep_period="$TIME_BASE" + else + local sleep_period=$(( RANDOM % TIME_RND + TIME_BASE )) + fi + sleep "$sleep_period" +} + +# FIRST thing is to create PID file +if [ ! -f "$WORKER_PID" ] ; then + echo "$$" > "$WORKER_PID" +else + echo "worker already running, exiting" + exit 1 +fi + +# XXX +# add checks for PID file every .1 seconds so sleep for x2 that just to +# make sure +sleep .2 + +while queue_work ; do + queue_shift + if [ ! -z "$queue_item" ] ; then + if [ ! -z "$CMD" ] ; then + $CMD $queue_item + else + eval "$queue_item" + fi + fi + sleep_pause +done + +# remove PID file right before exiting +rm -f "$WORKER_PID" -- cgit v1.2.3