cc/cirrus_work: bin/_atest3.sh annotate

annotate bin/_atest3.sh @ 383:def4c3375b02 plus

version of warc.py which uses igzip, therefore doesn't support the block position in the callback result

author	Henry S. Thompson <ht@inf.ed.ac.uk>
date	Thu, 11 Jun 2026 21:26:09 +0100
parents	a00e888e4df3
children

rev	line source
377 b3ba45af45c9 prep for exercise 4 Henry S. Thompson <ht@inf.ed.ac.uk> parents: diff changeset	1 #!/usr/bin/bash
b3ba45af45c9 prep for exercise 4 Henry S. Thompson <ht@inf.ed.ac.uk> parents: diff changeset	2 # Usage: _atest3.sh j n
379 a00e888e4df3 works now Henry S. Thompson <ht@inf.ed.ac.uk> parents: 377 diff changeset	3 # Process the jth group of cdx files taken n at a time
a00e888e4df3 works now Henry S. Thompson <ht@inf.ed.ac.uk> parents: 377 diff changeset	4 # with 10 parallel threads at a time
377 b3ba45af45c9 prep for exercise 4 Henry S. Thompson <ht@inf.ed.ac.uk> parents: diff changeset	5 j=$1
b3ba45af45c9 prep for exercise 4 Henry S. Thompson <ht@inf.ed.ac.uk> parents: diff changeset	6 n=$2
b3ba45af45c9 prep for exercise 4 Henry S. Thompson <ht@inf.ed.ac.uk> parents: diff changeset	7 h_s=$((n * j))
b3ba45af45c9 prep for exercise 4 Henry S. Thompson <ht@inf.ed.ac.uk> parents: diff changeset	8 h_e=$(( ( ( n * (j + 1) ) - 1 ) ))
b3ba45af45c9 prep for exercise 4 Henry S. Thompson <ht@inf.ed.ac.uk> parents: diff changeset	9 echo _a passed j=$j n=$n, processing "[$h_s,$h_e]"
b3ba45af45c9 prep for exercise 4 Henry S. Thompson <ht@inf.ed.ac.uk> parents: diff changeset	10 #seq --format "%03g" $h_s $h_e \| parallel -j 10 echo '$(date)' cdx-00'{}.gz' ' {#} \\| sus' #\| uniq_merge.py
b3ba45af45c9 prep for exercise 4 Henry S. Thompson <ht@inf.ed.ac.uk> parents: diff changeset	11 cd /work/dc007/dc007/hst/results/CC-MAIN-2025-33/warc_lmhx/idx
b3ba45af45c9 prep for exercise 4 Henry S. Thompson <ht@inf.ed.ac.uk> parents: diff changeset	12 seq --format "%03g" $h_s $h_e \| parallel -j 10 'uz cdx-00{}.gz \| /work/dc007/dc007/shared/bin/cdx2tsv.sh languages \| egrep -v "NA\|," \| sus' \| uniq_merge.py