Mercurial > hg > cc > cirrus_home

diff bin/doC2T.sh @ 163:ef961d91eea5

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
previous approach to lang/field extraction
author Henry S. Thompson <ht@inf.ed.ac.uk>
date Mon, 18 Jul 2022 18:16:27 +0100
parents
children
line wrap: on
line diff
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/bin/doC2T.sh	Mon Jul 18 18:16:27 2022 +0100
@@ -0,0 +1,10 @@
+#!/usr/bin/bash
+c=$1
+i=$2
+f=$(printf 'cdx-%05.0f.gz' $i)
+unpigz -dp 1 -c data/$c/cdx/warc/$f  | cdx2tsv.py '(filename,f.split("/",maxsplit=5)[4][0])' '(url,f.split(":",maxsplit=1)[0])' mime mime-detected languages > /dev/shm/tbl_${i}.raw.tsv 2> /dev/shm/tbl_${i}.errs
+#| tee /dev/shm/data_${i}
+#wc -l /dev/shm/data_${i} /dev/shm/tbl_${i}.raw.tsv
+mv /dev/shm/tbl_${i}.{errs,raw.tsv} results 
+#rm /dev/shm/data_${i}
+