#!/usr/bin/bash
# Usage: ex4_1.sh j n
# Process the jth group of cdx files taken n at a time
#   with 10 parallel threads at a time
j=$1
n=$2
c_s=$((n * j))
c_e=$(( ( ( n * (j + 1) ) - 1 ) ))
echo ex4_1 passed j=$j n=$n, processing "[$c_s,$c_e]"
cd /work/dc007/dc007/hst/results/CC-MAIN-2025-33/warc_lmhx/idx
seq --format "%03g" $c_s $c_e | parallel -j 10 \
                     /work/dc007/dc007/hst/bin/ex4_2.sh '{}' | uniq_merge.py