spark_your_data_center.sh

#!/bin/bash
#SBATCH --job-name=ows_preprocessing
#SBATCH --output=sparkjob-%j.out

set -x
set -e

## 1. Load modules
module load python

## 2. Create Spark cluster and run job
export SPATH="$(dirname "${SCRIPT_PATH}")"
${SPATH}/scripts/slurm_spark_cluster.sh

## 3. Clean up
# stop the workers
#scancel ${SLURM_JOBID}.0

# stop the master
#stop-master.sh
~