#!/usr/bin/bash
mkdir -p /dev/shm/hst

c=$1
i=$2
f=$(printf 'cdx-%05.0f.gz' $i)

unpigz -dp 1 -c data/$c/cdx/warc/$f  |  parallel --willcite --pipe -N 50000 -j10 "ix.py -x -h -c '/lustre/home/dc007/hst/bin/clm.sh /dev/shm/hst/'$i'.lmh_{#}.txt' 2>/dev/shm/hst/$i.lmh_{#}.errs"

cd /dev/shm/hst
tar -czf $HOME/results/${i}.lmh.tar.gz ${i}.lmh_*.{txt,errs}