view bin/do_idx.sh @ 101:e2e64c3d763e

bug4 fixed, but that created a new, earlier bug
author Henry S. Thompson <ht@inf.ed.ac.uk>
date Mon, 11 Sep 2023 22:06:45 +0100
parents
children 7d58dc01f329
line wrap: on
line source

#!/bin/bash
export res="$1"
orig="$2"
merge_date.py -d <(LC_ALL=C sort -m -k1,2 -s $res/ks_[0-9]*.tsv) $orig $res/idx # | \
exit
    tee /dev/stderr | \
          parallel -j 10 'echo {#} {} >$res/merge_{#}.log
                          echo $(date) {#} {}
                          export res
                          split -l 3000 --filter="igzip -c | \
                                                  tee >(wc -c >> \
                                                        $res/merge_{#}.log)" \
                                    {} > {}.gz && \
                                    rm {}'