# HG changeset patch
# User Henry S. Thompson <ht@inf.ed.ac.uk>
# Date 1781562707 -3600
# Node ID 615efac7fc0918d9841ca6ff716d9522393b4906
# Parent  7245d684b0b5a148e04336d4b925c5d735edf072
working for ex4a

diff -r 7245d684b0b5 -r 615efac7fc09 bin/ex4_1.sh
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/bin/ex4_1.sh	Mon Jun 15 23:31:47 2026 +0100
@@ -0,0 +1,14 @@
+#!/usr/bin/bash
+# Usage: ex4_1.sh j n
+# Process the jth group of cdx files taken n at a time
+#   with 10 parallel threads at a time
+j=$1
+n=$2
+c_s=$((n * j))
+c_e=$(( ( ( n * (j + 1) ) - 1 ) ))
+echo ex4_1 passed j=$j n=$n, processing "[$c_s,$c_e]"
+cd /work/dc007/dc007/hst/results/CC-MAIN-2025-33/warc_lmhx/idx
+seq --format "%03g" $c_s $c_e | parallel -j 10 \
+                     /work/dc007/dc007/hst/bin/ex4_2.sh '{}' | uniq_merge.py
+
+
diff -r 7245d684b0b5 -r 615efac7fc09 bin/ex4_2.sh
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/bin/ex4_2.sh	Mon Jun 15 23:31:47 2026 +0100
@@ -0,0 +1,3 @@
+#!/usr/bin/bash
+# Usage: ex4_2.sh [000-299]
+uz cdx-00${1}.gz | cdx2tsv.sh languages | egrep '^[^N]..$' | sus
diff -r 7245d684b0b5 -r 615efac7fc09 lib/python/cc/ex4.py
--- a/lib/python/cc/ex4.py	Fri Jun 12 16:14:55 2026 +0100
+++ b/lib/python/cc/ex4.py	Mon Jun 15 23:31:47 2026 +0100
@@ -1,3 +1,4 @@
+# cython: profile=False, language_level=3str
 '''Use this to get called by cdx2tsv.py'''
 
 from collections import defaultdict