Mercurial > hg > cc > cirrus_work

annotate bin/do_idx.sh @ 109:52c6a9b0fc8c

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
loosen must-match criterion in the both-messy case
author Henry Thompson <ht@markup.co.uk>
date Tue, 19 Sep 2023 19:29:41 +0100
parents 9403c02d5034
children a0ea1e4a714d
Ignore whitespace changes - Everywhere: Within whitespace: At end of lines:
rev   line source
101
e2e64c3d763e bug4 fixed, but that created a new, earlier bug
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
1 #!/bin/bash
e2e64c3d763e bug4 fixed, but that created a new, earlier bug
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
2 export res="$1"
e2e64c3d763e bug4 fixed, but that created a new, earlier bug
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
3 orig="$2"
105
9403c02d5034 switch to gzip -7 to get comparable compressed cdx block size
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 103
diff changeset
4 # igzip was faster, but produced bigger files, so went to gzip one step
9403c02d5034 switch to gzip -7 to get comparable compressed cdx block size
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 103
diff changeset
5 # smaller than default (-6), which produces slightly _smaller_ blocks.
103
7d58dc01f329 re-instate logging splits for .idx
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 101
diff changeset
6 merge_date.py <(LC_ALL=C sort -m -k1,2 -s $res/ks_[0-9]*.tsv) $orig $res/idx |\
101
e2e64c3d763e bug4 fixed, but that created a new, earlier bug
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
7 parallel -j 10 'echo {#} {} >$res/merge_{#}.log
e2e64c3d763e bug4 fixed, but that created a new, earlier bug
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
8 echo $(date) {#} {}
e2e64c3d763e bug4 fixed, but that created a new, earlier bug
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
9 export res
105
9403c02d5034 switch to gzip -7 to get comparable compressed cdx block size
Henry S. Thompson <ht@inf.ed.ac.uk>
parents: 103
diff changeset
10 split -l 3000 --filter="gzip -c -7 --keep | \
101
e2e64c3d763e bug4 fixed, but that created a new, earlier bug
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
11 tee >(wc -c >> \
e2e64c3d763e bug4 fixed, but that created a new, earlier bug
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
12 $res/merge_{#}.log)" \
e2e64c3d763e bug4 fixed, but that created a new, earlier bug
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
13 {} > {}.gz && \
e2e64c3d763e bug4 fixed, but that created a new, earlier bug
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
14 rm {}'