Mercurial > hg > cc > cirrus_work
comparison bin/do_idx.sh @ 116:5b952d16838c
a bit more logging
| author | Henry S. Thompson <ht@inf.ed.ac.uk> |
|---|---|
| date | Tue, 26 Sep 2023 18:55:11 +0100 |
| parents | a0ea1e4a714d |
| children | 0326805aa6df |
comparison
equal
deleted
inserted
replaced
| 115:0b1e6e134aca | 116:5b952d16838c |
|---|---|
| 6 done | 6 done |
| 7 export res="$1" | 7 export res="$1" |
| 8 orig="$2" | 8 orig="$2" |
| 9 # igzip was faster, but produced bigger files, so went to gzip one step | 9 # igzip was faster, but produced bigger files, so went to gzip one step |
| 10 # smaller than default (-6), which produces slightly _smaller_ blocks. | 10 # smaller than default (-6), which produces slightly _smaller_ blocks. |
| 11 merge_date.py $debug <(LC_ALL=C sort -m -k1,2 -s $res/ks_[0-9]*.tsv) $orig $res/idx |\ | 11 merge_date.py $debug <(LC_ALL=C sort -m -k1,3 -s $res/ks_[0-9]*.tsv) \ |
| 12 $orig $res/idx 2>$res/merge.log | \ | |
| 12 parallel -j 10 'echo {#} {} >$res/merge_{#}.log | 13 parallel -j 10 'echo {#} {} >$res/merge_{#}.log |
| 13 echo $(date) {#} {} | 14 echo $(date) {#} {} |
| 14 export res | 15 export res |
| 15 split -l 3000 --filter="gzip -c -7 --keep | \ | 16 split -l 3000 --filter="gzip -c -7 --keep | \ |
| 16 tee >(wc -c >> \ | 17 tee >(wc -c >> \ |