#!/usr/bin/bash
# Usage: _atest3.sh j n
# Process the jth group of cdx files taken n at a time
#   with 10 parallel threads at a time
j=$1
n=$2
h_s=$((n * j))
h_e=$(( ( ( n * (j + 1) ) - 1 ) ))
echo _a passed j=$j n=$n, processing "[$h_s,$h_e]"
#seq --format "%03g" $h_s $h_e | parallel -j 10 echo '$(date)' cdx-00'{}.gz' ' {#} \| sus' #| uniq_merge.py
cd /work/dc007/dc007/hst/results/CC-MAIN-2025-33/warc_lmhx/idx
seq --format "%03g" $h_s $h_e | parallel -j 10 'uz cdx-00{}.gz | /work/dc007/dc007/shared/bin/cdx2tsv.sh languages | egrep -v "NA|," | sus' | uniq_merge.py