view bin/doS2C.sh @ 190:f2bf736c2d40

working, with issues: 1) warc.export.cdx.path is not set 2) mime property isn't getting through 3) lastMod isn't showing up
author Henry S. Thompson <ht@inf.ed.ac.uk>
date Tue, 24 Sep 2024 17:08:05 +0100
parents 0072e4ee6c67
children
line wrap: on
line source

#!/usr/bin/bash
# Usage: doS2C.sh node task cc resdir segs workd dbfile
node=$1
task=$2
cc=$3
resdir=$4
segs=$5
workd=$6
dbfile=$7

f=${dbfile%.db}
i=${f#*cdx}

echo "> $node.$task: $segs $i"

sqlite3 $dbfile ".mode csv" ".once $workd/$i.csv" "select count(*),* from props group by segment,ftype,https,nlangs" ".quit"

echo "< $node.$task: $segs $i";