Mercurial > hg > cc > cirrus_home
annotate bin/ix.sh @ 103:d354e29c91a5
never should have added
| author | Henry S. Thompson <ht@inf.ed.ac.uk> |
|---|---|
| date | Mon, 19 Apr 2021 18:09:25 +0000 |
| parents | 0332076afc37 |
| children | baf56ff538f8 |
| rev | line source |
|---|---|
| 88 | 1 #!/usr/bin/bash |
|
92
d56465d5c51f
accepts index lines, less line-at-a-time
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
91
diff
changeset
|
2 # Extract records from warc files given length, offset and file triples |
| 88 | 3 # from stdin or as command line args |
|
93
4d870a7ec871
support a command to receive each result,
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
92
diff
changeset
|
4 # Usage [-d] [-w] [-h] [-b] [-e cmd] [ -x | length offset path ] |
|
92
d56465d5c51f
accepts index lines, less line-at-a-time
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
91
diff
changeset
|
5 # -d Debug output |
| 89 | 6 # -w WARC headers |
| 7 # -h HTTP headers | |
| 8 # -b HTTP body | |
|
92
d56465d5c51f
accepts index lines, less line-at-a-time
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
91
diff
changeset
|
9 # No switch defaults to whole record |
|
d56465d5c51f
accepts index lines, less line-at-a-time
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
91
diff
changeset
|
10 # -x take lines of from a cdx index file as input, extract triples |
|
93
4d870a7ec871
support a command to receive each result,
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
92
diff
changeset
|
11 # -e pipes each result thru cmd |
|
92
d56465d5c51f
accepts index lines, less line-at-a-time
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
91
diff
changeset
|
12 if [ "$1" = "-d" ] |
|
d56465d5c51f
accepts index lines, less line-at-a-time
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
91
diff
changeset
|
13 then |
|
d56465d5c51f
accepts index lines, less line-at-a-time
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
91
diff
changeset
|
14 d=1 |
|
102
0332076afc37
better dd error handling
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
93
diff
changeset
|
15 rm /tmp/ix_dd_log.txt /tmp/ix_triples.tsv |
|
92
d56465d5c51f
accepts index lines, less line-at-a-time
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
91
diff
changeset
|
16 shift |
|
d56465d5c51f
accepts index lines, less line-at-a-time
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
91
diff
changeset
|
17 fi |
| 89 | 18 if [ "$1" = "-w" ] |
| 19 then | |
| 20 shift | |
| 21 p=1 | |
| 22 w=1 | |
| 23 fi | |
| 24 if [ "$1" = "-h" ] | |
| 25 then | |
| 26 shift | |
| 27 p=1 | |
| 28 h=1 | |
| 29 fi | |
| 30 if [ "$1" = "-b" ] | |
| 31 then | |
| 32 shift | |
| 33 p=1 | |
| 34 b=1 | |
| 35 fi | |
|
93
4d870a7ec871
support a command to receive each result,
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
92
diff
changeset
|
36 e="cat" |
|
4d870a7ec871
support a command to receive each result,
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
92
diff
changeset
|
37 if [ "$1" = "-e" ] |
|
4d870a7ec871
support a command to receive each result,
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
92
diff
changeset
|
38 then |
|
4d870a7ec871
support a command to receive each result,
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
92
diff
changeset
|
39 shift |
|
4d870a7ec871
support a command to receive each result,
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
92
diff
changeset
|
40 e="$1" |
|
4d870a7ec871
support a command to receive each result,
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
92
diff
changeset
|
41 shift |
|
4d870a7ec871
support a command to receive each result,
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
92
diff
changeset
|
42 fi |
|
92
d56465d5c51f
accepts index lines, less line-at-a-time
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
91
diff
changeset
|
43 if [ "$1" = "-x" ] |
|
d56465d5c51f
accepts index lines, less line-at-a-time
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
91
diff
changeset
|
44 then |
|
d56465d5c51f
accepts index lines, less line-at-a-time
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
91
diff
changeset
|
45 # get triples from index lines |
|
d56465d5c51f
accepts index lines, less line-at-a-time
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
91
diff
changeset
|
46 egrep -ao 'length": "[0-9]*", "offset": "[0-9]*".*\.gz'| \ |
|
d56465d5c51f
accepts index lines, less line-at-a-time
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
91
diff
changeset
|
47 sed 's/[a-z]*": "//g;s/", "/\t/g;s/\(crawl-data\|segments\|warc\)\///g' |\ |
|
102
0332076afc37
better dd error handling
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
93
diff
changeset
|
48 if [ "$d" ] ; then tee -a /tmp/ix_triples.tsv ; else cat ; fi |
|
92
d56465d5c51f
accepts index lines, less line-at-a-time
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
91
diff
changeset
|
49 elif [ "$1" ] |
| 88 | 50 then |
| 51 printf "%s\t%s\t%s\n" "$1" "$2" "$3" | |
| 52 else | |
| 53 cat | |
| 54 fi | \ | |
|
92
d56465d5c51f
accepts index lines, less line-at-a-time
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
91
diff
changeset
|
55 while { IFS=$'\t' read l o f; } |
| 88 | 56 do |
|
92
d56465d5c51f
accepts index lines, less line-at-a-time
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
91
diff
changeset
|
57 if [ -z "$d" ] |
|
d56465d5c51f
accepts index lines, less line-at-a-time
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
91
diff
changeset
|
58 then |
|
d56465d5c51f
accepts index lines, less line-at-a-time
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
91
diff
changeset
|
59 dd if="$f" of=/dev/stdout skip=$o count=$l \ |
|
102
0332076afc37
better dd error handling
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
93
diff
changeset
|
60 iflag=skip_bytes,count_bytes status=none |
|
92
d56465d5c51f
accepts index lines, less line-at-a-time
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
91
diff
changeset
|
61 else |
|
d56465d5c51f
accepts index lines, less line-at-a-time
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
91
diff
changeset
|
62 echo dd if="$f" of=/dev/stdout skip=$o count=$l \ |
|
102
0332076afc37
better dd error handling
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
93
diff
changeset
|
63 iflag=skip_bytes,count_bytes >> /tmp/ix_dd_log.txt |
|
92
d56465d5c51f
accepts index lines, less line-at-a-time
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
91
diff
changeset
|
64 dd if="$f" of=/dev/stdout skip=$o count=$l \ |
|
d56465d5c51f
accepts index lines, less line-at-a-time
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
91
diff
changeset
|
65 iflag=skip_bytes,count_bytes 2>> /tmp/ix_dd_log.txt |
|
d56465d5c51f
accepts index lines, less line-at-a-time
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
91
diff
changeset
|
66 fi | \ |
|
102
0332076afc37
better dd error handling
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
93
diff
changeset
|
67 { unpigz -dp 1 -c || { printf "dd failure?: %s %s %s\n" $f $o $l 1>&2 ; continue ; } } |\ |
| 90 | 68 { s="w" |
|
92
d56465d5c51f
accepts index lines, less line-at-a-time
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
91
diff
changeset
|
69 if [ "$p" ] |
| 89 | 70 then |
| 91 | 71 shopt -qs extglob # for %%*(...) |
| 89 | 72 while read -r L |
| 73 do | |
| 74 if [ "$s" = "w" ] | |
| 75 then | |
| 76 # WARC header | |
|
92
d56465d5c51f
accepts index lines, less line-at-a-time
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
91
diff
changeset
|
77 case "$L" in |
|
d56465d5c51f
accepts index lines, less line-at-a-time
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
91
diff
changeset
|
78 Content-Length:\ *) wl=${L##*: } |
|
d56465d5c51f
accepts index lines, less line-at-a-time
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
91
diff
changeset
|
79 wl=${wl%%*([ [:space:]])} |
|
d56465d5c51f
accepts index lines, less line-at-a-time
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
91
diff
changeset
|
80 ;; |
|
d56465d5c51f
accepts index lines, less line-at-a-time
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
91
diff
changeset
|
81 WARC-Truncated:\ *) # echo $n $L |
|
d56465d5c51f
accepts index lines, less line-at-a-time
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
91
diff
changeset
|
82 tr=${L##*: } |
|
d56465d5c51f
accepts index lines, less line-at-a-time
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
91
diff
changeset
|
83 tr=${tr%%*([ [:space:]])} |
|
d56465d5c51f
accepts index lines, less line-at-a-time
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
91
diff
changeset
|
84 tr=${tr:-EMPTY} |
|
d56465d5c51f
accepts index lines, less line-at-a-time
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
91
diff
changeset
|
85 ;; |
|
d56465d5c51f
accepts index lines, less line-at-a-time
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
91
diff
changeset
|
86 ) s="h" |
|
d56465d5c51f
accepts index lines, less line-at-a-time
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
91
diff
changeset
|
87 if [ -z "$h$b" ] |
|
d56465d5c51f
accepts index lines, less line-at-a-time
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
91
diff
changeset
|
88 then |
|
d56465d5c51f
accepts index lines, less line-at-a-time
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
91
diff
changeset
|
89 exit 0 |
|
d56465d5c51f
accepts index lines, less line-at-a-time
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
91
diff
changeset
|
90 fi |
|
d56465d5c51f
accepts index lines, less line-at-a-time
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
91
diff
changeset
|
91 if [ "$w" ] |
|
d56465d5c51f
accepts index lines, less line-at-a-time
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
91
diff
changeset
|
92 then |
|
d56465d5c51f
accepts index lines, less line-at-a-time
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
91
diff
changeset
|
93 echo |
|
d56465d5c51f
accepts index lines, less line-at-a-time
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
91
diff
changeset
|
94 fi |
|
d56465d5c51f
accepts index lines, less line-at-a-time
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
91
diff
changeset
|
95 continue;; |
|
d56465d5c51f
accepts index lines, less line-at-a-time
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
91
diff
changeset
|
96 esac |
|
d56465d5c51f
accepts index lines, less line-at-a-time
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
91
diff
changeset
|
97 if [ "$w" ] |
| 89 | 98 then |
| 99 printf "%s\n" "${L%% }" | |
| 100 fi | |
| 101 continue | |
| 102 fi | |
| 90 | 103 if [ "$s" = "h" ] |
| 89 | 104 then |
| 105 # HTTP header | |
|
92
d56465d5c51f
accepts index lines, less line-at-a-time
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
91
diff
changeset
|
106 wl=$((wl - ( ${#L} + 1 ))) |
| 89 | 107 case "$L" in |
|
92
d56465d5c51f
accepts index lines, less line-at-a-time
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
91
diff
changeset
|
108 Content-Length:\ *) bl=${L##*: } |
|
d56465d5c51f
accepts index lines, less line-at-a-time
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
91
diff
changeset
|
109 bl=${bl%%*([ [:space:]])} |
|
d56465d5c51f
accepts index lines, less line-at-a-time
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
91
diff
changeset
|
110 ;; |
| 91 | 111 ) s="b" ; n=0 |
|
92
d56465d5c51f
accepts index lines, less line-at-a-time
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
91
diff
changeset
|
112 if [ -z "$b" ] |
|
d56465d5c51f
accepts index lines, less line-at-a-time
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
91
diff
changeset
|
113 then |
|
d56465d5c51f
accepts index lines, less line-at-a-time
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
91
diff
changeset
|
114 exit 0 |
|
d56465d5c51f
accepts index lines, less line-at-a-time
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
91
diff
changeset
|
115 fi |
|
d56465d5c51f
accepts index lines, less line-at-a-time
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
91
diff
changeset
|
116 if [ "$h" ] |
| 91 | 117 then |
| 118 echo | |
| 119 fi | |
|
92
d56465d5c51f
accepts index lines, less line-at-a-time
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
91
diff
changeset
|
120 if [ "$bl" ]; then |
|
d56465d5c51f
accepts index lines, less line-at-a-time
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
91
diff
changeset
|
121 if [ $bl -ne $wl ]; then |
|
93
4d870a7ec871
support a command to receive each result,
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
92
diff
changeset
|
122 echo length mismatch: $l $o $f here: $wl given: $bl trunc: $tr 1>&2 |
|
92
d56465d5c51f
accepts index lines, less line-at-a-time
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
91
diff
changeset
|
123 fi |
|
d56465d5c51f
accepts index lines, less line-at-a-time
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
91
diff
changeset
|
124 fi |
| 91 | 125 continue ;; |
| 89 | 126 esac |
|
92
d56465d5c51f
accepts index lines, less line-at-a-time
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
91
diff
changeset
|
127 if [ "$h" ] |
| 89 | 128 then |
| 129 printf "%s\n" "${L%% }" | |
| 130 fi | |
| 131 continue | |
| 132 else | |
| 133 # HTTP body | |
|
92
d56465d5c51f
accepts index lines, less line-at-a-time
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
91
diff
changeset
|
134 if [ "$b" ] |
| 89 | 135 then |
| 91 | 136 #printf "%s\n" "$bl" 1>&2 |
|
92
d56465d5c51f
accepts index lines, less line-at-a-time
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
91
diff
changeset
|
137 head -c "${bl-$wl}" |
|
d56465d5c51f
accepts index lines, less line-at-a-time
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
91
diff
changeset
|
138 exit 0 |
| 89 | 139 else |
| 140 break | |
| 141 fi | |
| 142 fi | |
| 143 done | |
| 144 else | |
| 90 | 145 # No flags,the whole thing |
| 146 cat | |
| 147 fi | |
|
93
4d870a7ec871
support a command to receive each result,
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
92
diff
changeset
|
148 } | $e |
|
92
d56465d5c51f
accepts index lines, less line-at-a-time
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
91
diff
changeset
|
149 done |
