Mercurial > hg > cc > cirrus_home

annotate bin/sql2csv.sh @ 152:0072e4ee6c67

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
use sqlite3 just to tabulate
author Henry S. Thompson <ht@inf.ed.ac.uk>
date Thu, 28 Oct 2021 12:11:08 +0000
parents
children
Ignore whitespace changes - Everywhere: Within whitespace: At end of lines:
rev   line source
152
0072e4ee6c67 use sqlite3 just to tabulate
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
1 #!/bin/bash
0072e4ee6c67 use sqlite3 just to tabulate
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
2 # Invoke this as e.g. sbatch -N 4 --ntasks=3 -c 5 masterJob.sh sql2csv \
0072e4ee6c67 use sqlite3 just to tabulate
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
3 # CC-MAIN-2019-35 cdx_db 20-43
0072e4ee6c67 use sqlite3 just to tabulate
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
4 # run sql2csv.py in parallel, taking input directly from .../24-43.n.c.tar.gz
0072e4ee6c67 use sqlite3 just to tabulate
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
5 n=$SLURM_NTASKS
0072e4ee6c67 use sqlite3 just to tabulate
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
6 c=$SLURM_CPUS_PER_TASK
0072e4ee6c67 use sqlite3 just to tabulate
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
7 node=$SLURMD_NODENAME
0072e4ee6c67 use sqlite3 just to tabulate
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
8 task=$SLURM_LOCALID
0072e4ee6c67 use sqlite3 just to tabulate
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
9 node=$SLURM_NODEID
0072e4ee6c67 use sqlite3 just to tabulate
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
10
0072e4ee6c67 use sqlite3 just to tabulate
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
11 cc=$1
0072e4ee6c67 use sqlite3 just to tabulate
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
12 resdir=$2
0072e4ee6c67 use sqlite3 just to tabulate
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
13 segs=$3
0072e4ee6c67 use sqlite3 just to tabulate
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
14
0072e4ee6c67 use sqlite3 just to tabulate
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
15 echo $(date) $nodename:$node:$task start
0072e4ee6c67 use sqlite3 just to tabulate
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
16
0072e4ee6c67 use sqlite3 just to tabulate
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
17 export PYTHONPATH=$PYTHONPATH:$HOME/lib/python
0072e4ee6c67 use sqlite3 just to tabulate
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
18
0072e4ee6c67 use sqlite3 just to tabulate
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
19 ld=/dev/shm/ht/$task
0072e4ee6c67 use sqlite3 just to tabulate
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
20 mkdir -p $ld
0072e4ee6c67 use sqlite3 just to tabulate
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
21
0072e4ee6c67 use sqlite3 just to tabulate
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
22 cd $ld
0072e4ee6c67 use sqlite3 just to tabulate
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
23 tar --wildcards -xf $HOME/results/$cc/$resdir/$segs.$node.$task.tar.gz '*.db'
0072e4ee6c67 use sqlite3 just to tabulate
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
24 cd $HOME
0072e4ee6c67 use sqlite3 just to tabulate
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
25
0072e4ee6c67 use sqlite3 just to tabulate
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
26 ls $ld/*.db | \
0072e4ee6c67 use sqlite3 just to tabulate
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
27 parallel --will-cite -j $c doS2C.sh $node $task $cc $resdir $segs $ld '{}'
0072e4ee6c67 use sqlite3 just to tabulate
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
28
0072e4ee6c67 use sqlite3 just to tabulate
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
29 if [ "$ld" ]
0072e4ee6c67 use sqlite3 just to tabulate
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
30 then
0072e4ee6c67 use sqlite3 just to tabulate
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
31 cd $ld
0072e4ee6c67 use sqlite3 just to tabulate
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
32 cp *.csv $HOME/results/$cc/$resdir
0072e4ee6c67 use sqlite3 just to tabulate
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
33 rm *
0072e4ee6c67 use sqlite3 just to tabulate
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
34 fi
0072e4ee6c67 use sqlite3 just to tabulate
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
35
0072e4ee6c67 use sqlite3 just to tabulate
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
36 echo $(date) $nodename:$node:$task end
0072e4ee6c67 use sqlite3 just to tabulate
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
37
0072e4ee6c67 use sqlite3 just to tabulate
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
38