Mercurial > hg > cc > cirrus_home

view bin/hawk.sh @ 79:4b8e4e3d60eb

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
track redirects, need to us full crawldiagnostics.warc.gz for "location:" and "Uri:"
author Henry S. Thompson <ht@inf.ed.ac.uk>
date Thu, 07 May 2020 18:47:24 +0100
parents 47ef882acbec
children
line wrap: on
line source

#!/bin/bash
# Usage: hawk.sh tarFileNumber
pdfs=/beegfs/common_crawl/CC-MAIN-2019-35/pdfs
links=$pdfs/links
dates=$pdfs/1132dates
tfn=$1
awk '{ if (NR == FNR) {
ll[$1]=1
}
else {
f="links_'${tfn}'_"$1
if (f in ll) {
print $0,"\t",f
}
}
}' <(tar -tf $links/$tfn.tar) ${dates}/date_$tfn.tsv > ${dates}/join_$tfn.tsv