#!/bin/bash
# Usage: hawk.sh tarFileNumber
pdfs=/beegfs/common_crawl/CC-MAIN-2019-35/pdfs
links=$pdfs/links
dates=$pdfs/1132dates
tfn=$1
awk '{ if (NR == FNR) {
ll[$1]=1
}
else {
f="links_'${tfn}'_"$1
if (f in ll) {
print $0,"\t",f
}
}
}' <(tar -tf $links/$tfn.tar) ${dates}/date_$tfn.tsv > ${dates}/join_$tfn.tsv
