Mercurial > hg > cc > cirrus_home

comparison bin/ix.sh @ 102:0332076afc37

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
better dd error handling
author Henry S. Thompson <ht@inf.ed.ac.uk>
date Mon, 19 Apr 2021 13:08:16 +0000
parents 4d870a7ec871
children baf56ff538f8
comparison
equal deleted inserted replaced
101:fbca56fabbac 102:0332076afc37
10 # -x take lines of from a cdx index file as input, extract triples 10 # -x take lines of from a cdx index file as input, extract triples
11 # -e pipes each result thru cmd 11 # -e pipes each result thru cmd
12 if [ "$1" = "-d" ] 12 if [ "$1" = "-d" ]
13 then 13 then
14 d=1 14 d=1
15 rm /tmp/ix_dd_log.txt /tmp/ix_triples.tsv
15 shift 16 shift
16 fi 17 fi
17 if [ "$1" = "-w" ] 18 if [ "$1" = "-w" ]
18 then 19 then
19 shift 20 shift
42 if [ "$1" = "-x" ] 43 if [ "$1" = "-x" ]
43 then 44 then
44 # get triples from index lines 45 # get triples from index lines
45 egrep -ao 'length": "[0-9]*", "offset": "[0-9]*".*\.gz'| \ 46 egrep -ao 'length": "[0-9]*", "offset": "[0-9]*".*\.gz'| \
46 sed 's/[a-z]*": "//g;s/", "/\t/g;s/\(crawl-data\|segments\|warc\)\///g' |\ 47 sed 's/[a-z]*": "//g;s/", "/\t/g;s/\(crawl-data\|segments\|warc\)\///g' |\
47 if [ "$d" ] ; then tee /tmp/ix_triples.tsv ; else cat ; fi 48 if [ "$d" ] ; then tee -a /tmp/ix_triples.tsv ; else cat ; fi
48 elif [ "$1" ] 49 elif [ "$1" ]
49 then 50 then
50 printf "%s\t%s\t%s\n" "$1" "$2" "$3" 51 printf "%s\t%s\t%s\n" "$1" "$2" "$3"
51 else 52 else
52 cat 53 cat
54 while { IFS=$'\t' read l o f; } 55 while { IFS=$'\t' read l o f; }
55 do 56 do
56 if [ -z "$d" ] 57 if [ -z "$d" ]
57 then 58 then
58 dd if="$f" of=/dev/stdout skip=$o count=$l \ 59 dd if="$f" of=/dev/stdout skip=$o count=$l \
59 iflag=skip_bytes,count_bytes 2>/dev/null 60 iflag=skip_bytes,count_bytes status=none
60 else 61 else
61 echo dd if="$f" of=/dev/stdout skip=$o count=$l \ 62 echo dd if="$f" of=/dev/stdout skip=$o count=$l \
62 iflag=skip_bytes,count_bytes > /tmp/ix_dd_log.txt 63 iflag=skip_bytes,count_bytes >> /tmp/ix_dd_log.txt
63 dd if="$f" of=/dev/stdout skip=$o count=$l \ 64 dd if="$f" of=/dev/stdout skip=$o count=$l \
64 iflag=skip_bytes,count_bytes 2>> /tmp/ix_dd_log.txt 65 iflag=skip_bytes,count_bytes 2>> /tmp/ix_dd_log.txt
65 fi | \ 66 fi | \
66 unpigz -dp 1 -c | tee /tmp/data | \ 67 { unpigz -dp 1 -c || { printf "dd failure?: %s %s %s\n" $f $o $l 1>&2 ; continue ; } } |\
67 { s="w" 68 { s="w"
68 if [ "$p" ] 69 if [ "$p" ]
69 then 70 then
70 shopt -qs extglob # for %%*(...) 71 shopt -qs extglob # for %%*(...)
71 while read -r L 72 while read -r L