Mercurial > hg > cc > cirrus_home

annotate bin/preExtract.sh @ 45:bd0010ac88ce

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
parallelised version of reExtract.sh
author Henry S. Thompson <ht@inf.ed.ac.uk>
date Wed, 08 Apr 2020 11:27:33 +0100
parents
children 2e5b3439a2ed
Ignore whitespace changes - Everywhere: Within whitespace: At end of lines:
rev   line source
45
bd0010ac88ce parallelised version of reExtract.sh
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
1 #!/usr/bin/bash
bd0010ac88ce parallelised version of reExtract.sh
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
2 # Usage: reExtract.sh 20..-.. < files...
bd0010ac88ce parallelised version of reExtract.sh
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
3 # Input is list of paths to relative path of warc files
bd0010ac88ce parallelised version of reExtract.sh
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
4 # under /beegfs/common_crawl/CC-MAIN-$1
bd0010ac88ce parallelised version of reExtract.sh
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
5
bd0010ac88ce parallelised version of reExtract.sh
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
6 function sus ()
bd0010ac88ce parallelised version of reExtract.sh
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
7 {
bd0010ac88ce parallelised version of reExtract.sh
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
8 sort "$@" | uniq -c | sort -k1nr,1
bd0010ac88ce parallelised version of reExtract.sh
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
9 }
bd0010ac88ce parallelised version of reExtract.sh
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
10
bd0010ac88ce parallelised version of reExtract.sh
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
11 function edex () {
bd0010ac88ce parallelised version of reExtract.sh
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
12 echo $(($(fgrep -n $1 ~/by11n.txt | cut -f 1 -d :) - 1))
bd0010ac88ce parallelised version of reExtract.sh
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
13 }
bd0010ac88ce parallelised version of reExtract.sh
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
14
bd0010ac88ce parallelised version of reExtract.sh
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
15 function join_by () {
bd0010ac88ce parallelised version of reExtract.sh
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
16 # Courtesy of https://stackoverflow.com/a/17841619/2595465
bd0010ac88ce parallelised version of reExtract.sh
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
17 local d=$1
bd0010ac88ce parallelised version of reExtract.sh
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
18 shift
bd0010ac88ce parallelised version of reExtract.sh
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
19 echo -n "$1"
bd0010ac88ce parallelised version of reExtract.sh
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
20 shift
bd0010ac88ce parallelised version of reExtract.sh
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
21 printf "%s" "${@/#/$d}"
bd0010ac88ce parallelised version of reExtract.sh
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
22 }
bd0010ac88ce parallelised version of reExtract.sh
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
23
bd0010ac88ce parallelised version of reExtract.sh
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
24 h=/beegfs/common_crawl/CC-MAIN-$1
bd0010ac88ce parallelised version of reExtract.sh
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
25
bd0010ac88ce parallelised version of reExtract.sh
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
26 mkdir -p /dev/shm/rex
bd0010ac88ce parallelised version of reExtract.sh
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
27 cd /dev/shm/rex
bd0010ac88ce parallelised version of reExtract.sh
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
28
bd0010ac88ce parallelised version of reExtract.sh
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
29 sed 's/-00/ /;s/.warc.gz//' | \
bd0010ac88ce parallelised version of reExtract.sh
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
30 while read s p i
bd0010ac88ce parallelised version of reExtract.sh
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
31 do
bd0010ac88ce parallelised version of reExtract.sh
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
32 printf "%s\t%s\t%s\t%s\n" $(edex $i) $s $p $i
bd0010ac88ce parallelised version of reExtract.sh
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
33 done > in.txt
bd0010ac88ce parallelised version of reExtract.sh
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
34 cut -f 2,3 in.txt| sort -u | tr '\011' '\012' |\
bd0010ac88ce parallelised version of reExtract.sh
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
35 parallel --will-cite -j 16 -N 2 h="$h"'
bd0010ac88ce parallelised version of reExtract.sh
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
36 function sus () { sort "$@" | uniq -c | sort -k1nr,1 ; }
bd0010ac88ce parallelised version of reExtract.sh
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
37 s={1}
bd0010ac88ce parallelised version of reExtract.sh
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
38 p={2}
bd0010ac88ce parallelised version of reExtract.sh
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
39 mkdir -p $s/logs
bd0010ac88ce parallelised version of reExtract.sh
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
40 cd $s
bd0010ac88ce parallelised version of reExtract.sh
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
41 # Sigh, should not have used this in the extraction ...
bd0010ac88ce parallelised version of reExtract.sh
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
42 jobid=$(tar -tf ${h}/${s}/extract_0.tar logs/ | head -1 | \
bd0010ac88ce parallelised version of reExtract.sh
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
43 cut -f 2 -d / |cut -f 1 -d _)
bd0010ac88ce parallelised version of reExtract.sh
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
44 for e in $(egrep "\b$s\b" ../in.txt |cut -f 1 | sort -u)
bd0010ac88ce parallelised version of reExtract.sh
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
45 # this could be parallel
bd0010ac88ce parallelised version of reExtract.sh
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
46 do
bd0010ac88ce parallelised version of reExtract.sh
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
47 lsf=lsl${e}.txt
bd0010ac88ce parallelised version of reExtract.sh
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
48 rm -f $lsf
bd0010ac88ce parallelised version of reExtract.sh
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
49 lff=()
bd0010ac88ce parallelised version of reExtract.sh
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
50 ii=()
bd0010ac88ce parallelised version of reExtract.sh
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
51 for i in $(egrep "^$e\\s$s\\b" ../in.txt|cut -f 4)
bd0010ac88ce parallelised version of reExtract.sh
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
52 # this could be parallel
bd0010ac88ce parallelised version of reExtract.sh
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
53 do
bd0010ac88ce parallelised version of reExtract.sh
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
54 id=${p#CC-MAIN-*}-00$i
bd0010ac88ce parallelised version of reExtract.sh
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
55 echo "$id" 1>&2
bd0010ac88ce parallelised version of reExtract.sh
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
56 lf=logs/${jobid}_${i}_log
bd0010ac88ce parallelised version of reExtract.sh
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
57 lff+=("${lf}") # accumulate list of log files
bd0010ac88ce parallelised version of reExtract.sh
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
58 echo starting ${id} $(date) > $lf
bd0010ac88ce parallelised version of reExtract.sh
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
59 unpigz -dp 1 -c ${h}/${s}/CC-MAIN-${id}.warc.gz | $HOME/lib/valhalla/bin/warc.sh ${id} application/pdf 2>> $lf
bd0010ac88ce parallelised version of reExtract.sh
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
60 echo finished ${id} $(date) >> $lf
bd0010ac88ce parallelised version of reExtract.sh
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
61 ls -l ${id}_* | tr -s " " "\t" |cut -f 5,9 >> $lsf
bd0010ac88ce parallelised version of reExtract.sh
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
62 ii+=("*-00${i}_*")
bd0010ac88ce parallelised version of reExtract.sh
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
63 echo "$i" "${ii[@]}" ${#ii[@]} 1>&2
bd0010ac88ce parallelised version of reExtract.sh
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
64 done
bd0010ac88ce parallelised version of reExtract.sh
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
65 # now compare ls vs. tar
bd0010ac88ce parallelised version of reExtract.sh
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
66 echo "${ii[@]}" ${#ii[@]} 1>&2
bd0010ac88ce parallelised version of reExtract.sh
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
67 echo lff "${lff[@]}" 1>&2
bd0010ac88ce parallelised version of reExtract.sh
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
68 tar -tvf ${h}/${s}/extract_${e}.tar "${ii[@]}" | \
bd0010ac88ce parallelised version of reExtract.sh
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
69 tr -s " " "\t" |cut -f 3,6 |sort -k2.1,2.36 -k2.38,2n | diff -bw \
bd0010ac88ce parallelised version of reExtract.sh
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
70 - <(sort -k2.1,2.36 -k2.38,2n $lsf) > ${e}_diff.txt
bd0010ac88ce parallelised version of reExtract.sh
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
71 cut -f 1 -d " " ${e}_diff.txt | sus > ${e}_check.txt
bd0010ac88ce parallelised version of reExtract.sh
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
72 ni=${#ii[@]}
bd0010ac88ce parallelised version of reExtract.sh
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
73 if [ $(wc -l < ${e}_check.txt) -ne $((ni + 1)) ]
bd0010ac88ce parallelised version of reExtract.sh
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
74 then
bd0010ac88ce parallelised version of reExtract.sh
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
75 echo "extra lines in ${e}_check.txt" 1>&2
bd0010ac88ce parallelised version of reExtract.sh
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
76 cd ..
bd0010ac88ce parallelised version of reExtract.sh
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
77 break
bd0010ac88ce parallelised version of reExtract.sh
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
78 fi
bd0010ac88ce parallelised version of reExtract.sh
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
79 if [ $(fgrep -c a ${e}_check.txt) -ne $ni ]
bd0010ac88ce parallelised version of reExtract.sh
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
80 then
bd0010ac88ce parallelised version of reExtract.sh
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
81 echo "non-addition lines in ${e}_check.txt" 1>&2
bd0010ac88ce parallelised version of reExtract.sh
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
82 cd ..
bd0010ac88ce parallelised version of reExtract.sh
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
83 break
bd0010ac88ce parallelised version of reExtract.sh
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
84 fi
bd0010ac88ce parallelised version of reExtract.sh
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
85 egrep "^> " ${e}_diff.txt | cut -f 2 > ${e}_new.txt
bd0010ac88ce parallelised version of reExtract.sh
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
86 tar --delete -f ${h}/${s}/extract_${e}.tar "${lff[@]}"
bd0010ac88ce parallelised version of reExtract.sh
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
87 tar --append -f ${h}/${s}/extract_${e}.tar --files-from=${e}_new.txt "${lff[@]}"
bd0010ac88ce parallelised version of reExtract.sh
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
88 done
bd0010ac88ce parallelised version of reExtract.sh
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
89 cd ..
bd0010ac88ce parallelised version of reExtract.sh
Henry S. Thompson <ht@inf.ed.ac.uk>
parents:
diff changeset
90 '