Mercurial > hg > cc > work

changeset 90:82d406eca22b

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
x
author Henry S. Thompson <ht@inf.ed.ac.uk>
date Fri, 09 May 2025 17:38:38 +0100
parents 367dca1afea4
children b9a79d52e2d0
files lurid3/notes.txt
diffstat 1 files changed, 224 insertions(+), 0 deletions(-) [+]
line wrap: on
line diff
--- a/lurid3/notes.txt	Fri May 02 17:09:22 2025 +0100
+++ b/lurid3/notes.txt	Fri May 09 17:38:38 2025 +0100
@@ -3772,3 +3772,227 @@
   cdbmake: fatal: unable to create lmh.tmp: out of memory
 
 Sigh.  So, 5 at a time it is.
+  >: { cat {0..4}/*.cdb_in ; echo ; } > lmh_0-4.cdb_in
+  >: wc -l lmh_0-4.cdb_in
+  29019908 lmh_0-4.cdb_in
+  >: cdbmake lmh_0-4.cdb lmh.tmp < lmh_0-4.cdb_in
+  [worked]
+  >: cdbstats <lmh_0-4.cdb
+  records   29006541
+  d0        21752329
+  d1         4163904
+  d2         1493895
+  d3          684077
+  d4          357679
+  d5          202932
+  d6          121301
+  d7           75556
+  d8           49004
+  d9           32275
+  >9           73589
+  >: cdbtest <lmh_0-4.cdb
+  found: 29005457
+  different record: 1084
+  bad length: 0
+  not found: 0
+  untested: 13366
+  >: cdbget 20190825142846http://71.43.189.10/dermorph/ <lmh_0-4.cdb
+  1564555978u
+  >: head -10000 2/lmh.cdb_in|cut -f 2- -d : | rev | cut -c 13- | rev > /tmp/hst/10000keys
+  >: time while read k; do cdbget "$k" < lmh_0-4.cdb; echo; done < /tmp/hst/10000keys 2>/dev/null | wc -l
+  10000
+
+  real    0m18.252s
+  user    0m6.954s
+  sys     0m10.213s
+
+Ah, but that "different record" above signals a problem...
+  >: cat /tmp/hst/clipped 
+  >: cut -f 2- -d : lmh_0-4.cdb_in | rev | cut -c 13- | rev|sus | head -2000 | tee /tmp/hst/dups_maybe | head -30
+     5 20190505175536https://www.iadb.org/robots.txt
+     5 20190513133933https://brokercheck.finra.org/robots.txt
+     5 20190601161509http://d3fy651gv2fhd3.cloudfront.net/robots.txt
+     5 20190602064638https://solarsystem.nasa.gov/robots.txt
+     5 20190603072038https://static-cdn.hungaricana.hu/static/library/robots.txt
+     ...
+  >: tail -1 /tmp/hst/dups_maybe
+     2 20190817182112http://www.libreriadelgiurista.it/reparti/libri/riviste-giuridiche/rivista-di-diritto-f
+So not all robots.txt :-(
+  >: fgrep -c robots.txt /tmp/hst/dups_maybe
+  794
+  >: wc -l /tmp/hst/dups_maybe
+  813 /tmp/hst/dups_maybe
+Oops -- the head -30 killed its predecessor early :-(
+Values?
+>: cdbget 20190812091100https://www.afd.fr/robots.txt < lmh_0-4.cdb
+1541675798
+>: cdbget 20190812091100https://www.afd.fr/robots.txt 1 < lmh_0-4.cdb
+1541675798
+>: cdbget 20190812091100https://www.afd.fr/robots.txt 2 < lmh_0-4.cdb
+1541675798
+>: cdbget 20190812091100https://www.afd.fr/robots.txt 3 < lmh_0-4.cdb
+1541675798
+>: cdbget 20190812091100https://www.afd.fr/robots.txt 4 < lmh_0-4.cdb
+[nothing]
+  >: tr -s ' ' '\t'< /tmp/hst/dups_maybe |cut -f 2 |btot
+  1861
+  >: fgrep -v robots.txt /tmp/hst/dups_maybe |head
+	3 20190817102328https://gloria.tv/robots.tx
+	2 20190507094924http://contentz.mkt922.com/lp/static/notfound.html
+	2 20190815142304https://www.digikey.es/robots_ssl.txt
+	2 20190816102602https://usbeketrica.com/robots.tx
+	2 20190816123029https://s-ryhma.fi/robots.tx
+	2 20190817051518http://www.cnrmz.com/cnrmz_404/404.html
+	2 20190817075606http://cdlcommercial.com.sg
+	2 20190817102328https://www.gloria.tv/robots.tx
+	2 20190817103745https://results.gothiacup.se/2019/start
+	2 20190817181648http://www.libreriadelgiurista.it/reparti/libri/diritto-penale/parte-speciale/delitti-contro-lamministr/simulazione-di-reato/
+
+This one looks worrying:
+ https://results.gothiacup.se/2019/start
+
+See email to ccrawl about all this on 2025-05-05
+
+Redo the check:
+  >: cut -f 2- -d : lmh_0-4.cdb_in | rev | cut -c 13- | rev|sus | head -2000 > /tmp/hst/dups_maybe
+  >: head -882 /tmp/hst/dups_maybe|tail -2
+      2 20200524222757https://www.ledtronics.com/robots.txt
+  >: fgrep -n ' 1 ' /tmp/hst/dups_maybe | head -1
+  882:      1
+  >: head -881 /tmp/hst/dups_maybe| acut 2 | fgrep -vi robots.txt |cut -f 3 -d / | sus
+     29 www.libreriadelgiurista.it
+      1 cdlcommercial.com.sg
+      1 contentz.mkt922.com
+      1 gloria.tv
+      1 m.kekenet.com
+      1 results.gothiacup.se
+      1 s-ryhma.fi
+      1 usbeketrica.com
+      1 www.beatstars.com
+      1 www.cnrmz.com
+      1 www.digikey.es
+      1 www.gloria.tv
+      1 www.nordprofi
+  >: uz /beegfs/common_crawl/CC-MAIN-2019-35/cdx/warc/cdx-00208.gz | egrep '^it,libreriadelgiurista\)' |  wc -l
+  7504
+  >: uz /beegfs/common_crawl/CC-MAIN-2019-35/cdx/warc/cdx-00208.gz | egrep -c '^it,libreriadelgiurista\)/robots.txt'
+  104
+  >: uz /beegfs/common_crawl/CC-MAIN-2019-35/cdx/warc/cdx-00208.gz | egrep '^it,libreriadelgiurista\)' |  egrep -c /robotstxt/
+  104
+
+Looks like _maybe_ the segment number will disambiguate...  Patched
+warc2cdb to try that.  Seems to work, rebuild first decile to check...
+  >: for ty in robotstxt; do sbatch --output=slurm_0-9x5${ty::1}_n.out --time=00:10:00 --ntasks=10 -c 15 --exclusive $HOME/bin/runme.sh $PWD -d -t 3 -i 'cat ranges' 'export PYTHONPATH=$W/hst/lib/python/cc:$W/hst/lib/python/cc/lmh
+j=$((i - 1)) ; tp=$((task)) ; ty='$ty'
+mkdir -p ./x$j/$tp/$ty
+~/lib/python/cc/w2c.py 2019-35 $tp ./x$j $ty "$arg"  2> ./x$j/$tp/$ty/w2c_errs'; done
+
+  >: for s in {0..4}; do (mkdir -p $s; cd $s; for i in {0..4}; do for t in warc robotstxt crawldiagnostics; do head -n $(($(wc -l < ../../x$i/$s/$t/lmh.cdb_in) - 1))  ../../x$i/$s/$t/lmh.cdb_in >> lmh.cdb_in; done ; done); done
+
+Make the cdb
+  >: { cat {0..4}/*.cdb_in ; echo ; } > lmh_0-4.cdb_in
+  >: cdbmake lmh_0-4.cdb lmh.tmp < lmh_0-4.cdb_in
+  >: wc -l lmh_0-4.cdb_in
+  29019908 lmh_0-4.cdb_in
+  >: cdbtest < lmh_0-4.cdb
+  found: 29006541
+  different record: 0
+  bad length: 0
+  not found: 0
+  untested: 13366
+
+But still a problem?
+  >: grep '20190817181648http://www.libreriadelgiurista.it/reparti/libri/diritto-penale/parte-speciale/delitti-contro-lamministr/simulazione-di-reato/' lmh_0-4.cdb_in
+  +139,10:20190817181648http://www.libreriadelgiurista.it/reparti/libri/diritto-penale/parte-speciale/delitti-contro-lamministr/simulazione-di-reato/->1566152208
+  +148,1:20190817181648http://www.libreriadelgiurista.it/reparti/libri/diritto-penale/parte-speciale/delitti-contro-lamministr/simulazione-di-reato/2634.html->0
+Problem is with my checking code :-(
+Fixed:
+ >: cut -f 2- -d : lmh_0-4.cdb_in| sed 's/\(.*\)->/\1/' |sus | head -2000 > /tmp/hst/dups_maybe_3
+  >: head /tmp/hst/dups_maybe_3
+      1
+      1 197002010310023http://www.posow.org/robots.txt1336984352
+      1 197002151706150http://80.242.35.37:8082/robots.txt1105372083 
+OK, we're good.
+
+Could make the index smaller by removing fixed substrings, e.g.
+ 201908 (see status.xlsx above)
+ http
+ :  -- no: tiny number with :/, one with only :, in lmh_0-4.cdb_in
+
+>: wc< lmh_0-4.cdb_in
+  29,019,908   31,259,494 3,233,730,936
+
+Would removing 10chars per line make a difference?
+Rebuild 5/lmh.cdb_in, then try
+
+  >: wc */lmh.cdb_in
+   5909613    6367560  658513822 0/lmh.cdb_in
+   5828354    6273393  648392929 1/lmh.cdb_in
+   5720231    6158851  636531231 2/lmh.cdb_in
+   5727497    6161609  637764972 3/lmh.cdb_in
+   5834212    6298081  652527981 4/lmh.cdb_in
+   5856535    6323050  652761362 5/lmh.cdb_in
+  34,876,442   37,582,544 3,886,492,297 total
+  >: echo $((3886492297: - (34876442 * 10)))
+  3,537,727,877
+Probably not.
+Ah, but
+  >: egrep -ce '->156.......$' lmh_0-5.cdb_in
+  27737244
+That's 80% of the values...
+so encoding ->156 as ->0 would save another 2 * 27M:
+  >: echo $((3886492297 - ((34876442 * 10)+(27737244 * 2))))
+  3,482,253,389
+Let's try that.
+  >: cd mcdb
+  >: wc lmh_0-5.cdb_in
+  34876443   34876442 3440541052
+
+Worked!
+  : wc lmh_0-5.cdb_in
+    34876443   34876442 3440541052 lmh_0-5.cdb_incirrus-login1<6047>: cat /tmp/hst/clipped 
+  >: cdbmake lmh_0-5.cdb lmh.tmp < lmh_0-5.cdb_in
+  >: cdbtest < lmh_0-5.cdb
+  found: 34860624
+  different record: 0
+  bad length: 0
+  not found: 0
+  untested: 15818
+  >: cdbget 120911000s://www.afd.fr/robots.txt < lmh_0-5.cdb
+  1541675798
+
+  >: cut -f 2- -d : lmh_0-5.cdb_in| sed 's/\(.*\)->/\1/' |sus | head -2000 > /tmp/hst/dups_maybe_4
+  >: less /tmp/hst/dups_maybe_4
+  >: head -2 /tmp/hst/dups_maybe_4
+	1
+	1 010200543://chuangye.yjbys.com/robots.txt1523845314
+
+All good.
+
+As well, we could re-order the segments by size of cdb, to see if we
+could find enough 6-seg so that overall we could get to fewer cdbs
+overall
+
+First, rebuild 10..99
+  ...
+  >: for ty in warc; do sbatch --output=slurm_80-99x5${ty::1}_n.out --time=02:00:00 --ntasks=20 -c 15 $HOME/bin/runme.sh $PWD -d -t 3 -i 'cat ranges' 'export PYTHONPATH=$W/hst/lib/python/cc:$W/hst/lib/python/cc/lmh
+j=$((i - 1)) ; tp=$((task + 80)) ; ty='$ty'
+mkdir -p ./x$j/$tp/$ty
+~/lib/python/cc/w2c.py 2019-35 $tp ./x$j $ty "$arg" 156 201908 2> ./x$j/$tp/$ty/w2c_errs'; done
+  >: for ty in crawldiagnostics robotstxt; do sbatch --output=slurm_80-99x5${ty::1}_n.out --time=00:30:00 --ntasks=20 -c 15 $HOME/bin/runme.sh $PWD -d -t 3 -i 'cat ranges' 'export PYTHONPATH=$W/hst/lib/python/cc:$W/hst/lib/python/cc/lmh
+j=$((i - 1)) ; tp=$((task + 80)) ; ty='$ty'
+mkdir -p ./x$j/$tp/$ty
+~/lib/python/cc/w2c.py 2019-35 $tp ./x$j $ty "$arg" 156 201908 2> ./x$j/$tp/$ty/w2c_errs'; done
+  >: cd mcdb
+  >: seq 80 99 | parallel -j 5 's={}; mkdir -p $s; cd $s; for i in {0..4}; do for t in warc robotstxt crawldiagnostics; do head -n $(($(wc -l < ../../x$i/$s/$t/lmh.cdb_in) - 1))  ../../x$i/$s/$t/lmh.cdb_in >> lmh.cdb_in; done ; done'
+  >: seq 78 6 89 | parallel -j 2 's={}; e=$((s + 5)); { eval cat $(echo {$s..$e}/*.cdb_in) ; echo ; } > lmh_$s-$e.cdb_in'
+  >: seq 90 5 99 | parallel -j 2 's={}; e=$((s + 4)); { eval cat $(echo {$s..$e}/*.cdb_in) ; echo ; } > lmh_$s-$e.cdb_in'
+  >: seq 78 6 89 | parallel -j 2 's={}; e=$((s + 5)); cdbmake lmh_$s-$e.cdb lmh$s.tmp < lmh_$s-$e.cdb_in'
+  >: seq 90 5 99 | parallel -j 2 's={}; e=$((s + 4)); cdbmake lmh_$s-$e.cdb lmh$s.tmp < lmh_$s-$e.cdb_in'
+  >: { seq 78 6 89 ; seq 90 5 99 ; } | parallel -j 2 's={}; cdbtest < lmh_$s-??.cdb'
+  found: 34970132
+  different record: 0
+  bad length: 0
+  not found: 0
+  untested: 15277
+  ...