Mercurial > hg > cc > cirrus_home

annotate bin/clmp.sh @ 175:d123ef7fdb82

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
working on implementing types and parts: 1, 2, 4 working, 3 not
author Henry S. Thompson <ht@inf.ed.ac.uk>
date Mon, 03 Jul 2023 18:16:14 +0100
parents ef961d91eea5
children
Ignore whitespace changes - Everywhere: Within whitespace: At end of lines:
rev   line source
163
ef961d91eea5 previous approach to lang/field extraction
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
1 #!/bin/bash
ef961d91eea5 previous approach to lang/field extraction
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
2 # Invoke this as e.g. sbatch --time=5:00:00 --exclusive -N 5 --ntasks-per-node 2 -c 1 masterJob.sh clmp CC-MAIN-2019-35 cdx_tab1
ef961d91eea5 previous approach to lang/field extraction
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
3 # run clm.py in parallel
ef961d91eea5 previous approach to lang/field extraction
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
4 n=$SLURM_NTASKS
ef961d91eea5 previous approach to lang/field extraction
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
5 c=$SLURM_CPUS_PER_TASK
ef961d91eea5 previous approach to lang/field extraction
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
6 nodename=$SLURMD_NODENAME
ef961d91eea5 previous approach to lang/field extraction
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
7 local=$SLURM_LOCALID
ef961d91eea5 previous approach to lang/field extraction
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
8 node=$SLURM_NODEID
ef961d91eea5 previous approach to lang/field extraction
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
9 echo $(date) $nodename:$node start
ef961d91eea5 previous approach to lang/field extraction
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
10
ef961d91eea5 previous approach to lang/field extraction
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
11 type parallel
ef961d91eea5 previous approach to lang/field extraction
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
12 #module load gnu-parallel
ef961d91eea5 previous approach to lang/field extraction
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
13
ef961d91eea5 previous approach to lang/field extraction
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
14 export PYTHONPATH=$PYTHONPATH:$HOME/lib/python
ef961d91eea5 previous approach to lang/field extraction
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
15 parallel --will-cite -j $c doCLM.sh "$1" '{}' < $2/$node.txt
ef961d91eea5 previous approach to lang/field extraction
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
16
ef961d91eea5 previous approach to lang/field extraction
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
17 echo $(date) $nodename:$node end