|
3
|
1 # General-purpose environment variables and utility functions
|
|
|
2 # Recommended for inclusion in ${HOME}/.bashrc via
|
|
|
3 # source /work/dc007/dc007/shared/.bash_extras
|
|
|
4 # Already sourced in singularity container _before_ bash is launched
|
|
|
5 # Note that $HOME (and therefore the meaning of ~ in paths) is
|
|
|
6 # set to $W in the singularity container for convenience, as it is
|
|
|
7 # assumed to be where you will spend most of your time
|
|
|
8
|
|
|
9 H=/home/dc007/dc007/${USER} # Only visible from login nodes
|
|
|
10 W=/work/dc007/dc007/${USER}
|
|
|
11
|
|
|
12 CCC=/work/dc007/shared/pod12/common_crawl
|
|
|
13 SHARED=/work/dc007/dc007/shared
|
|
|
14
|
|
|
15 PATH=$PATH:$HOME/.local/bin:$HOME/bin:$W/bin:$SHARED/bin
|
|
|
16
|
|
|
17 export H W CCC SHARED PATH
|
|
1
|
18
|
|
|
19 cu ()
|
|
|
20 {
|
|
|
21 cvs update "$@" | egrep -v '^\?'
|
|
|
22 }
|
|
|
23
|
|
|
24 lss ()
|
|
|
25 {
|
|
|
26 ls -F --color=auto -lS "$@" | {
|
|
|
27 read one;
|
|
|
28 if [ -n "$one" -a "${one::5}" \!= "total" ]; then
|
|
|
29 echo "$one";
|
|
|
30 fi;
|
|
|
31 cat
|
|
|
32 }
|
|
|
33 }
|
|
|
34
|
|
|
35 ff ()
|
|
|
36 {
|
|
|
37 pat=$1;
|
|
|
38 shift;
|
|
|
39 find . -iname "*$pat*" $@
|
|
|
40 }
|
|
|
41
|
|
|
42 typecat ()
|
|
|
43 {
|
|
|
44 tt=$(type $1);
|
|
|
45 case $tt in
|
|
|
46 *\ is\ hashed\ *)
|
|
|
47 cat $(echo $tt | cut -f 4 -d ' ' | tr -d '()')
|
|
|
48 ;;
|
|
|
49 *\ is\ /*)
|
|
|
50 cat $(echo $tt | cut -f 3 -d ' ' | tr -d '()')
|
|
|
51 ;;
|
|
|
52 *)
|
|
|
53 type $1
|
|
|
54 ;;
|
|
|
55 esac
|
|
|
56 }
|
|
|
57
|
|
|
58 sus ()
|
|
|
59 {
|
|
|
60 sort "$@" | uniq -c | sort -k1nr,1
|
|
|
61 }
|
|
|
62
|
|
|
63 uz ()
|
|
|
64 {
|
|
3
|
65 igzip -dc "$@" # only works under sing
|
|
1
|
66 }
|
|
|
67
|
|
|
68 function tot ()
|
|
|
69 {
|
|
|
70 awk '{sum+=$1} END {printf "%u\n",sum}'
|
|
|
71 }
|
|
|
72
|
|
|
73 btot ()
|
|
|
74 {
|
|
|
75 python3 -c 'import sys
|
|
|
76 multiCol = dostats = sample = False
|
|
|
77 c = ""
|
|
|
78 p = 0
|
|
|
79 q = None
|
|
|
80 d = 2
|
|
|
81 ssq = 0
|
|
|
82 while len(sys.argv)>1:
|
|
|
83 if multiCol and sys.argv[1][0]!="-":
|
|
|
84 p=1
|
|
|
85 q=int(sys.argv.pop(1))
|
|
|
86 elif sys.argv[1]=="-c":
|
|
|
87 # add commas
|
|
|
88 c = ","
|
|
|
89 sys.argv.pop(1)
|
|
|
90 elif sys.argv[1]=="-a":
|
|
|
91 # do sum etc. for all columns [p]:q
|
|
|
92 multiCol = True
|
|
|
93 sys.argv.pop(1)
|
|
|
94 try:
|
|
|
95 q = int(sys.argv.pop(1))
|
|
|
96 except IndexError:
|
|
|
97 print("you must supply an N for multi-column")
|
|
|
98 exit(1)
|
|
|
99 elif sys.argv[1]=="-s":
|
|
|
100 # include mean and sample stdev
|
|
|
101 dostats = True
|
|
|
102 sample = True
|
|
|
103 sys.argv.pop(1)
|
|
|
104 elif sys.argv[1]=="-p":
|
|
|
105 # include mean and population stdev
|
|
|
106 dostats = True
|
|
|
107 sys.argv.pop(1)
|
|
|
108 elif sys.argv[1]=="-d":
|
|
|
109 sys.argv.pop(1)
|
|
|
110 d=int(sys.argv.pop(1))
|
|
|
111 else:
|
|
|
112 break
|
|
|
113 n=0
|
|
|
114 if multiCol:
|
|
|
115 w=q-p
|
|
|
116 import numpy as np
|
|
|
117 s=np.array([0]*w)
|
|
|
118 n=np.array([0]*w)
|
|
|
119 if dostats:
|
|
|
120 ssq=np.array([0]*w)
|
|
|
121 else:
|
|
|
122 s=0
|
|
|
123 if dostats:
|
|
|
124 ssq=0
|
|
|
125 import math
|
|
|
126 for l in sys.stdin: # ["1 11\n","2 22\n"]:
|
|
|
127 try:
|
|
|
128 if multiCol:
|
|
|
129 ll=l.split()[p:q]
|
|
|
130 ii=np.array([(0 if i.startswith("NaN") else int(i)) for i in ll])
|
|
|
131 s+=ii
|
|
|
132 ssq+=ii*ii
|
|
|
133 n+=np.array([(0 if i.startswith("NaN") else 1) for i in ll])
|
|
|
134 else:
|
|
|
135 if l[0]!="N":
|
|
|
136 i=int(l)
|
|
|
137 s+=i
|
|
|
138 if dostats:
|
|
|
139 ssq+=(i*i)
|
|
|
140 n+=1
|
|
|
141 except (NameError,ValueError) as e:
|
|
|
142 print(l,e,file=sys.stderr)
|
|
|
143 if dostats:
|
|
|
144 if multiCol:
|
|
|
145 m=s/n
|
|
|
146 var=(ssq/n)-pow(m,2)
|
|
|
147 if sample:
|
|
|
148 var=(n/(n-1))*var
|
|
|
149 sd=pow(var,0.5)
|
|
|
150 else:
|
|
|
151 m=float(s)/float(n)
|
|
|
152 var=(float(ssq)/float(n))-pow(float(m),2)
|
|
|
153 if sample:
|
|
|
154 var=(float(n)/float(n-1))*var
|
|
|
155 sd=math.sqrt(var)
|
|
|
156 if dostats:
|
|
|
157 if multiCol:
|
|
|
158 print("\t".join(f"{i:{c}.{d}f}" for i in s))
|
|
|
159 print("\t".join(f"{i:{c}.{d}f}" for i in m))
|
|
|
160 print("\t".join(f"{i:{c}.{d}f}" for i in sd))
|
|
|
161 else:
|
|
|
162 print(f"{s:}\t{m:}\t{sd:}")
|
|
|
163 else:
|
|
|
164 if multiCol:
|
|
|
165 print("\t".join(f"{i:}" for i in s))
|
|
|
166 else:
|
|
|
167 print(f"{s:}")
|
|
|
168
|
|
|
169 ' "$@"
|
|
|
170 }
|
|
|
171
|
|
|
172 headtail ()
|
|
|
173 {
|
|
|
174 case "$1" in -*)
|
|
|
175 n="$1"
|
|
|
176 shift
|
|
|
177 ;;
|
|
|
178 *) n="-10"
|
|
|
179 ;;
|
|
|
180 esac
|
|
|
181 cat "$@" | { sed -u ${n:1}q ; echo === ; tail $n ; }
|
|
|
182 }
|
|
|
183
|
|
|
184 export -f cu lss ff typecat sus uz tot btot headtail
|
|
7
|
185
|
|
|
186
|
|
|
187
|
|
|
188
|
|
|
189
|
|
|
190
|
