Mercurial > hg > cc > cirrus_work
annotate bin/do_idx.sh @ 101:e2e64c3d763e
bug4 fixed, but that created a new, earlier bug
author | Henry S. Thompson <ht@inf.ed.ac.uk> |
---|---|
date | Mon, 11 Sep 2023 22:06:45 +0100 |
parents | |
children | 7d58dc01f329 |
rev | line source |
---|---|
101
e2e64c3d763e
bug4 fixed, but that created a new, earlier bug
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff
changeset
|
1 #!/bin/bash |
e2e64c3d763e
bug4 fixed, but that created a new, earlier bug
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff
changeset
|
2 export res="$1" |
e2e64c3d763e
bug4 fixed, but that created a new, earlier bug
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff
changeset
|
3 orig="$2" |
e2e64c3d763e
bug4 fixed, but that created a new, earlier bug
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff
changeset
|
4 merge_date.py -d <(LC_ALL=C sort -m -k1,2 -s $res/ks_[0-9]*.tsv) $orig $res/idx # | \ |
e2e64c3d763e
bug4 fixed, but that created a new, earlier bug
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff
changeset
|
5 exit |
e2e64c3d763e
bug4 fixed, but that created a new, earlier bug
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff
changeset
|
6 tee /dev/stderr | \ |
e2e64c3d763e
bug4 fixed, but that created a new, earlier bug
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff
changeset
|
7 parallel -j 10 'echo {#} {} >$res/merge_{#}.log |
e2e64c3d763e
bug4 fixed, but that created a new, earlier bug
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff
changeset
|
8 echo $(date) {#} {} |
e2e64c3d763e
bug4 fixed, but that created a new, earlier bug
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff
changeset
|
9 export res |
e2e64c3d763e
bug4 fixed, but that created a new, earlier bug
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff
changeset
|
10 split -l 3000 --filter="igzip -c | \ |
e2e64c3d763e
bug4 fixed, but that created a new, earlier bug
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff
changeset
|
11 tee >(wc -c >> \ |
e2e64c3d763e
bug4 fixed, but that created a new, earlier bug
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff
changeset
|
12 $res/merge_{#}.log)" \ |
e2e64c3d763e
bug4 fixed, but that created a new, earlier bug
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff
changeset
|
13 {} > {}.gz && \ |
e2e64c3d763e
bug4 fixed, but that created a new, earlier bug
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff
changeset
|
14 rm {}' |