191
|
1 <?xml version="1.0"?>
|
|
2 <!--
|
|
3 Licensed to the Apache Software Foundation (ASF) under one or more
|
|
4 contributor license agreements. See the NOTICE file distributed with
|
|
5 this work for additional information regarding copyright ownership.
|
|
6 The ASF licenses this file to You under the Apache License, Version 2.0
|
|
7 (the "License"); you may not use this file except in compliance with
|
|
8 the License. You may obtain a copy of the License at
|
|
9
|
|
10 http://www.apache.org/licenses/LICENSE-2.0
|
|
11
|
|
12 Unless required by applicable law or agreed to in writing, software
|
|
13 distributed under the License is distributed on an "AS IS" BASIS,
|
|
14 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
15 See the License for the specific language governing permissions and
|
|
16 limitations under the License.
|
|
17 -->
|
|
18 <project name="${name}" default="runtime" xmlns:ivy="antlib:org.apache.ivy.ant" xmlns:artifact="antlib:org.apache.maven.artifact.ant">
|
|
19
|
|
20 <!-- Load all the default properties, and any the user wants -->
|
|
21 <!-- to contribute (without having to type -D or edit this file -->
|
|
22 <property file="${user.home}/build.properties" />
|
|
23 <property file="${basedir}/build.properties" />
|
|
24 <property file="${basedir}/default.properties" />
|
|
25 <property name="release.dir" value="${build.dir}/release"/>
|
|
26
|
|
27 <!-- define Maven coordinates, repository url and artifacts name etc -->
|
|
28 <property name="groupId" value="org.apache.nutch" />
|
|
29 <property name="artifactId" value="nutch" />
|
|
30 <property name="maven-repository-url" value="https://repository.apache.org/service/local/staging/deploy/maven2" />
|
|
31 <property name="maven-repository-id" value="apache.releases.https" />
|
|
32 <property name="maven-jar" value="${release.dir}/${artifactId}-${version}.jar" />
|
|
33 <property name="maven-javadoc-jar" value="${release.dir}/${artifactId}-${version}-javadoc.jar" />
|
|
34 <property name="maven-sources-jar" value="${release.dir}/${artifactId}-${version}-sources.jar" />
|
|
35
|
|
36
|
|
37 <!-- the normal classpath -->
|
|
38 <path id="classpath">
|
|
39 <pathelement location="${build.classes}"/>
|
|
40 <fileset dir="${build.lib.dir}">
|
|
41 <include name="*.jar" />
|
|
42 </fileset>
|
|
43 </path>
|
|
44
|
|
45 <dirname property="plugins.classpath.dir" file="${build.plugins}"/>
|
|
46
|
|
47 <!-- the unit test classpath -->
|
|
48 <path id="test.classpath">
|
|
49 <pathelement location="${test.build.classes}" />
|
|
50 <pathelement location="${conf.dir}"/>
|
|
51 <pathelement location="${test.src.dir}"/>
|
|
52 <pathelement location="${plugins.classpath.dir}"/>
|
|
53 <path refid="classpath"/>
|
|
54 <pathelement location="${build.dir}/${final.name}.job" />
|
|
55 <fileset dir="${build.lib.dir}">
|
|
56 <include name="*.jar" />
|
|
57 </fileset>
|
|
58 <fileset dir="${test.build.lib.dir}">
|
|
59 <include name="*.jar" />
|
|
60 </fileset>
|
|
61 </path>
|
|
62
|
|
63 <presetdef name="javac">
|
|
64 <javac includeantruntime="false" />
|
|
65 </presetdef>
|
|
66
|
|
67 <!-- ====================================================== -->
|
|
68 <!-- Stuff needed by all targets -->
|
|
69 <!-- ====================================================== -->
|
|
70 <target name="init" depends="ivy-init" description="--> stuff required by all targets">
|
|
71 <mkdir dir="${build.dir}"/>
|
|
72 <mkdir dir="${build.classes}"/>
|
|
73 <mkdir dir="${release.dir}"/>
|
|
74
|
|
75 <mkdir dir="${test.build.dir}"/>
|
|
76 <mkdir dir="${test.build.classes}"/>
|
|
77 <mkdir dir="${test.build.lib.dir}"/>
|
|
78
|
|
79 <touch datetime="01/25/1971 2:00 pm">
|
|
80 <fileset dir="${conf.dir}" includes="**/*.template"/>
|
|
81 </touch>
|
|
82
|
|
83 <copy todir="${conf.dir}" verbose="true">
|
|
84 <fileset dir="${conf.dir}" includes="**/*.template"/>
|
|
85 <mapper type="glob" from="*.template" to="*"/>
|
|
86 </copy>
|
|
87 </target>
|
|
88
|
|
89 <!-- ====================================================== -->
|
|
90 <!-- Compile the Java files -->
|
|
91 <!-- ====================================================== -->
|
|
92 <target name="compile" depends="compile-core, compile-plugins" description="--> compile all Java files"/>
|
|
93
|
|
94 <target name="compile-core" depends="init, resolve-default" description="--> compile core Java files only">
|
|
95 <javac
|
|
96 encoding="${build.encoding}"
|
|
97 srcdir="${src.dir}"
|
|
98 includes="org/apache/nutch/**/*.java org/commoncrawl/**/*.java"
|
|
99 destdir="${build.classes}"
|
|
100 debug="${javac.debug}"
|
|
101 optimize="${javac.optimize}"
|
|
102 target="${javac.version}"
|
|
103 source="${javac.version}"
|
|
104 deprecation="${javac.deprecation}">
|
|
105 <compilerarg value="-Xlint:-path"/>
|
|
106 <classpath refid="classpath"/>
|
|
107 </javac>
|
|
108 <copy todir="${build.classes}">
|
|
109 <fileset dir="${src.dir}" includes="**/*.html" />
|
|
110 <fileset dir="${src.dir}" includes="**/*.css" />
|
|
111 <fileset dir="${src.dir}" includes="**/*.properties" />
|
|
112 </copy>
|
|
113 </target>
|
|
114
|
|
115 <target name="compile-plugins" depends="init, resolve-default" description="--> compile plugins only">
|
|
116 <ant dir="src/plugin" target="deploy" inheritAll="false"/>
|
|
117 </target>
|
|
118
|
|
119 <!-- ================================================================== -->
|
|
120 <!-- Make nutch.jar -->
|
|
121 <!-- ================================================================== -->
|
|
122 <!-- -->
|
|
123 <!-- ================================================================== -->
|
|
124 <target name="jar" depends="compile-core" description="--> make nutch.jar">
|
|
125 <copy file="${conf.dir}/nutch-default.xml"
|
|
126 todir="${build.classes}"/>
|
|
127 <copy file="${conf.dir}/nutch-site.xml"
|
|
128 todir="${build.classes}"/>
|
|
129 <jar jarfile="${build.dir}/${final.name}.jar"
|
|
130 basedir="${build.classes}">
|
|
131 <manifest>
|
|
132 </manifest>
|
|
133 </jar>
|
|
134 </target>
|
|
135
|
|
136 <!-- ================================================================== -->
|
|
137 <!-- Make Maven Central Release -->
|
|
138 <!-- ================================================================== -->
|
|
139 <!-- -->
|
|
140 <!-- ================================================================== -->
|
|
141 <target name="release" depends="compile-core" description="--> generate the release distribution">
|
|
142 <copy file="${conf.dir}/nutch-default.xml"
|
|
143 todir="${build.classes}"/>
|
|
144 <copy file="${conf.dir}/nutch-site.xml"
|
|
145 todir="${build.classes}"/>
|
|
146
|
|
147 <!-- build the main artifact -->
|
|
148 <jar jarfile="${maven-jar}" basedir="${build.classes}" />
|
|
149
|
|
150 <fail message="Unsupported Java version: ${java.version}. Javadoc requires Java version 7u25 or greater. See https://issues.apache.org/jira/browse/NUTCH-1590">
|
|
151 <condition>
|
|
152 <or>
|
|
153 <matches string="${java.version}" pattern="1.7.0_2[01234].+" casesensitive="false" />
|
|
154 <matches string="${java.version}" pattern="1.7.0_1.+" casesensitive="false" />
|
|
155 <equals arg1="${ant.java.version}" arg2="1.6" />
|
|
156 <equals arg1="${ant.java.version}" arg2="1.5" />
|
|
157 </or>
|
|
158 </condition>
|
|
159 </fail>
|
|
160
|
|
161 <!-- build the javadoc artifact -->
|
|
162 <javadoc
|
|
163 destdir="${release.dir}/javadoc"
|
|
164 overview="${src.dir}/overview.html"
|
|
165 author="true"
|
|
166 version="true"
|
|
167 use="true"
|
|
168 windowtitle="${name} ${version} API"
|
|
169 doctitle="${name} ${version} API"
|
|
170 bottom="Copyright &copy; ${year} The Apache Software Foundation"
|
|
171 >
|
|
172 <arg value="${javadoc.proxy.host}"/>
|
|
173 <arg value="${javadoc.proxy.port}"/>
|
|
174
|
|
175 <packageset dir="${src.dir}"/>
|
|
176 <packageset dir="${plugins.dir}/any23/src/java/" />
|
|
177 <packageset dir="${plugins.dir}/creativecommons/src/java"/>
|
|
178 <packageset dir="${plugins.dir}/feed/src/java"/>
|
|
179 <packageset dir="${plugins.dir}/headings/src/java"/>
|
|
180 <packageset dir="${plugins.dir}/exchange-jexl/src/java"/>
|
|
181 <packageset dir="${plugins.dir}/index-anchor/src/java"/>
|
|
182 <packageset dir="${plugins.dir}/index-basic/src/java"/>
|
|
183 <packageset dir="${plugins.dir}/index-geoip/src/java"/>
|
|
184 <packageset dir="${plugins.dir}/index-jexl-filter/src/java"/>
|
|
185 <packageset dir="${plugins.dir}/index-links/src/java"/>
|
|
186 <packageset dir="${plugins.dir}/index-metadata/src/java"/>
|
|
187 <packageset dir="${plugins.dir}/index-more/src/java"/>
|
|
188 <packageset dir="${plugins.dir}/index-replace/src/java"/>
|
|
189 <packageset dir="${plugins.dir}/index-static/src/java"/>
|
|
190 <packageset dir="${plugins.dir}/indexer-cloudsearch/src/java/" />
|
|
191 <packageset dir="${plugins.dir}/indexer-csv/src/java"/>
|
|
192 <packageset dir="${plugins.dir}/indexer-dummy/src/java"/>
|
|
193 <packageset dir="${plugins.dir}/indexer-elastic-rest/src/java/"/>
|
|
194 <packageset dir="${plugins.dir}/indexer-elastic/src/java/" />
|
|
195 <packageset dir="${plugins.dir}/indexer-kafka/src/java/" />
|
|
196 <packageset dir="${plugins.dir}/indexer-rabbit/src/java"/>
|
|
197 <packageset dir="${plugins.dir}/indexer-solr/src/java"/>
|
|
198 <packageset dir="${plugins.dir}/language-identifier/src/java"/>
|
|
199 <packageset dir="${plugins.dir}/lib-htmlunit/src/java"/>
|
|
200 <packageset dir="${plugins.dir}/lib-http/src/java"/>
|
|
201 <packageset dir="${plugins.dir}/lib-rabbitmq/src/java"/>
|
|
202 <packageset dir="${plugins.dir}/lib-regex-filter/src/java"/>
|
|
203 <packageset dir="${plugins.dir}/lib-selenium/src/java"/>
|
|
204 <packageset dir="${plugins.dir}/microformats-reltag/src/java"/>
|
|
205 <packageset dir="${plugins.dir}/mimetype-filter/src/java"/>
|
|
206 <packageset dir="${plugins.dir}/parse-ext/src/java"/>
|
|
207 <packageset dir="${plugins.dir}/parse-html/src/java"/>
|
|
208 <packageset dir="${plugins.dir}/parse-js/src/java"/>
|
|
209 <packageset dir="${plugins.dir}/parse-metatags/src/java"/>
|
|
210 <packageset dir="${plugins.dir}/parse-swf/src/java"/>
|
|
211 <packageset dir="${plugins.dir}/parse-tika/src/java"/>
|
|
212 <packageset dir="${plugins.dir}/parse-zip/src/java"/>
|
|
213 <packageset dir="${plugins.dir}/parsefilter-naivebayes/src/java"/>
|
|
214 <packageset dir="${plugins.dir}/parsefilter-regex/src/java"/>
|
|
215 <packageset dir="${plugins.dir}/protocol-file/src/java"/>
|
|
216 <packageset dir="${plugins.dir}/protocol-ftp/src/java"/>
|
|
217 <packageset dir="${plugins.dir}/protocol-htmlunit/src/java"/>
|
|
218 <packageset dir="${plugins.dir}/protocol-http/src/java"/>
|
|
219 <packageset dir="${plugins.dir}/protocol-httpclient/src/java"/>
|
|
220 <packageset dir="${plugins.dir}/protocol-interactiveselenium/src/java"/>
|
|
221 <packageset dir="${plugins.dir}/protocol-okhttp/src/java"/>
|
|
222 <packageset dir="${plugins.dir}/protocol-selenium/src/java"/>
|
|
223 <packageset dir="${plugins.dir}/publish-rabbitmq/src/java"/>
|
|
224 <packageset dir="${plugins.dir}/scoring-adaptive/src/java"/>
|
|
225 <packageset dir="${plugins.dir}/scoring-depth/src/java"/>
|
|
226 <packageset dir="${plugins.dir}/scoring-link/src/java"/>
|
|
227 <packageset dir="${plugins.dir}/scoring-opic/src/java"/>
|
|
228 <packageset dir="${plugins.dir}/scoring-orphan/src/java"/>
|
|
229 <packageset dir="${plugins.dir}/scoring-similarity/src/java"/>
|
|
230 <packageset dir="${plugins.dir}/subcollection/src/java"/>
|
|
231 <packageset dir="${plugins.dir}/tld/src/java"/>
|
|
232 <packageset dir="${plugins.dir}/urlfilter-automaton/src/java"/>
|
|
233 <packageset dir="${plugins.dir}/urlfilter-domain/src/java"/>
|
|
234 <packageset dir="${plugins.dir}/urlfilter-domainblacklist/src/java"/>
|
|
235 <packageset dir="${plugins.dir}/urlfilter-fast/src/java"/>
|
|
236 <packageset dir="${plugins.dir}/urlfilter-ignoreexempt/src/java"/>
|
|
237 <packageset dir="${plugins.dir}/urlfilter-prefix/src/java"/>
|
|
238 <packageset dir="${plugins.dir}/urlfilter-regex/src/java"/>
|
|
239 <packageset dir="${plugins.dir}/urlfilter-suffix/src/java"/>
|
|
240 <packageset dir="${plugins.dir}/urlfilter-validator/src/java"/>
|
|
241 <packageset dir="${plugins.dir}/urlmeta/src/java"/>
|
|
242 <packageset dir="${plugins.dir}/urlnormalizer-ajax/src/java"/>
|
|
243 <packageset dir="${plugins.dir}/urlnormalizer-basic/src/java"/>
|
|
244 <packageset dir="${plugins.dir}/urlnormalizer-host/src/java"/>
|
|
245 <packageset dir="${plugins.dir}/urlnormalizer-pass/src/java"/>
|
|
246 <packageset dir="${plugins.dir}/urlnormalizer-protocol/src/java"/>
|
|
247 <packageset dir="${plugins.dir}/urlnormalizer-querystring/src/java"/>
|
|
248 <packageset dir="${plugins.dir}/urlnormalizer-regex/src/java"/>
|
|
249 <packageset dir="${plugins.dir}/urlnormalizer-slash/src/java"/>
|
|
250
|
|
251 <link href="${javadoc.link.java}"/>
|
|
252 <link href="${javadoc.link.hadoop}"/>
|
|
253 <link href="${javadoc.link.lucene.core}"/>
|
|
254 <link href="${javadoc.link.lucene.analyzers-common}"/>
|
|
255 <link href="${javadoc.link.solr-solrj}"/>
|
|
256
|
|
257 <classpath refid="classpath"/>
|
|
258 <classpath>
|
|
259 <fileset dir="${plugins.dir}" >
|
|
260 <include name="**/*.jar"/>
|
|
261 </fileset>
|
|
262 </classpath>
|
|
263
|
|
264 <group title="Core" packages="org.apache.nutch.*"/>
|
|
265 <group title="Plugins API" packages="${plugins.api}"/>
|
|
266 <group title="Protocol Plugins" packages="${plugins.protocol}"/>
|
|
267 <group title="URL Filter Plugins" packages="${plugins.urlfilter}"/>
|
|
268 <group title="URL Normalizer Plugins" packages="${plugins.urlnormalizer}"/>
|
|
269 <group title="Scoring Plugins" packages="${plugins.scoring}"/>
|
|
270 <group title="Parse Plugins" packages="${plugins.parse}"/>
|
|
271 <group title="Parse Filter Plugins" packages="${plugins.parsefilter}"/>
|
|
272 <group title="Publisher Plugins" packages="${plugins.publisher}"/>
|
|
273 <group title="Exchange Plugins" packages="${plugins.exchange}"/>
|
|
274 <group title="Indexing Filter Plugins" packages="${plugins.index}"/>
|
|
275 <group title="Indexer Plugins" packages="${plugins.indexer}"/>
|
|
276 <group title="Misc. Plugins" packages="${plugins.misc}"/>
|
|
277 </javadoc>
|
|
278 <jar jarfile="${maven-javadoc-jar}">
|
|
279 <fileset dir="${release.dir}/javadoc" />
|
|
280 </jar>
|
|
281
|
|
282 <!-- build the sources artifact -->
|
|
283 <jar jarfile="${maven-sources-jar}">
|
|
284 <fileset dir="${src.dir}" />
|
|
285 </jar>
|
|
286 </target>
|
|
287
|
|
288 <!-- ================================================================== -->
|
|
289 <!-- Deploy to Apache Nexus -->
|
|
290 <!-- ================================================================== -->
|
|
291 <!-- -->
|
|
292 <!-- ================================================================== -->
|
|
293 <target name="deploy" depends="release" description="--> deploy to Apache Nexus">
|
|
294
|
|
295 <!-- generate a pom file -->
|
|
296 <ivy:makepom ivyfile="${ivy.file}" pomfile="${basedir}/pom.xml" templatefile="ivy/mvn.template">
|
|
297 <mapping conf="default" scope="compile"/>
|
|
298 <mapping conf="runtime" scope="runtime"/>
|
|
299 </ivy:makepom>
|
|
300
|
|
301 <!-- sign and deploy the main artifact -->
|
|
302 <artifact:mvn>
|
|
303 <arg value="org.apache.maven.plugins:maven-gpg-plugin:1.5:sign-and-deploy-file" />
|
|
304 <arg value="-Durl=${maven-repository-url}" />
|
|
305 <arg value="-DrepositoryId=${maven-repository-id}" />
|
|
306 <arg value="-DpomFile=pom.xml" />
|
|
307 <arg value="-Dfile=${maven-jar}" />
|
|
308 <arg value="-Papache-release" />
|
|
309 </artifact:mvn>
|
|
310
|
|
311 <!-- sign and deploy the sources artifact -->
|
|
312 <artifact:mvn>
|
|
313 <arg value="org.apache.maven.plugins:maven-gpg-plugin:1.5:sign-and-deploy-file" />
|
|
314 <arg value="-Durl=${maven-repository-url}" />
|
|
315 <arg value="-DrepositoryId=${maven-repository-id}" />
|
|
316 <arg value="-DpomFile=pom.xml" />
|
|
317 <arg value="-Dfile=${maven-sources-jar}" />
|
|
318 <arg value="-Dclassifier=sources" />
|
|
319 <arg value="-Papache-release" />
|
|
320 </artifact:mvn>
|
|
321
|
|
322 <!-- sign and deploy the javadoc artifact -->
|
|
323 <artifact:mvn>
|
|
324 <arg value="org.apache.maven.plugins:maven-gpg-plugin:1.5:sign-and-deploy-file" />
|
|
325 <arg value="-Durl=${maven-repository-url}" />
|
|
326 <arg value="-DrepositoryId=${maven-repository-id}" />
|
|
327 <arg value="-DpomFile=pom.xml" />
|
|
328 <arg value="-Dfile=${maven-javadoc-jar}" />
|
|
329 <arg value="-Dclassifier=javadoc" />
|
|
330 <arg value="-Papache-release" />
|
|
331 </artifact:mvn>
|
|
332 </target>
|
|
333
|
|
334 <!-- ================================================================== -->
|
|
335 <!-- Generate REST API Documentation with Miredot -->
|
|
336 <!-- ================================================================== -->
|
|
337 <target name="restdocs" description="--> generate REST API Documentation with Miredot">
|
|
338
|
|
339 <!-- generate a pom file -->
|
|
340 <ivy:makepom ivyfile="${ivy.file}" pomfile="${basedir}/pom.xml" templatefile="ivy/mvn.template">
|
|
341 <mapping conf="default" scope="compile"/>
|
|
342 <mapping conf="runtime" scope="runtime"/>
|
|
343 </ivy:makepom>
|
|
344
|
|
345 <!--artifact:dependencies pathId="dependency.classpath">
|
|
346 <dependency groupId="log4j" artifactId="log4j" version="1.2.15" >
|
|
347 <exclusion groupId="javax.jms" artifactId="jms" />
|
|
348 <exclusion groupId="com.sun.jdmk" artifactId="jmxtools" />
|
|
349 <exclusion groupId="com.sun.jmx" artifactId="jmxri" />
|
|
350 </dependency>
|
|
351 </artifact:dependencies-->
|
|
352
|
|
353 <artifact:mvn>
|
|
354 <arg value="test"/>
|
|
355 <arg value="-e"/>
|
|
356 <arg value="-o"/>
|
|
357 <!-- run offline (-o): must not download dependencies as this is
|
|
358 done from http://repo1.maven.org/ hardwired in
|
|
359 maven-ant-tasks-2.1.3.jar, see NUTCH-2722.
|
|
360
|
|
361 Dependencies and plugins need to be resolved and cached locally beforehand
|
|
362 by running
|
|
363 `mvn dependency:resolve`
|
|
364 resp.
|
|
365 `mvn dependency:resolve-plugins`
|
|
366 after the pom.xml has been generated. -->
|
|
367 </artifact:mvn>
|
|
368 </target>
|
|
369
|
|
370 <!-- ================================================================== -->
|
|
371 <!-- Make job jar -->
|
|
372 <!-- ================================================================== -->
|
|
373 <!-- -->
|
|
374 <!-- ================================================================== -->
|
|
375 <target name="job" depends="compile" description="--> make nutch.job jar">
|
|
376 <jar jarfile="${build.dir}/${final.name}.job">
|
|
377 <!-- If the build.classes has the nutch config files because the jar
|
|
378 command command has run, exclude them. The conf directory has
|
|
379 them.
|
|
380 -->
|
|
381 <zipfileset dir="${build.classes}"
|
|
382 excludes="nutch-default.xml,nutch-site.xml"/>
|
|
383 <zipfileset dir="${conf.dir}" excludes="*.template,hadoop*.*"/>
|
|
384 <zipfileset dir="${build.lib.dir}" prefix="lib"
|
|
385 includes="**/*.jar" excludes="hadoop-*.jar,slf4j*.jar,log4j*.jar"/>
|
|
386 <zipfileset dir="${build.plugins}" prefix="classes/plugins"/>
|
|
387 </jar>
|
|
388 </target>
|
|
389
|
|
390 <target name="runtime" depends="jar, job" description="--> default target for running Nutch">
|
|
391 <mkdir dir="${runtime.dir}"/>
|
|
392 <mkdir dir="${runtime.local}"/>
|
|
393 <mkdir dir="${runtime.deploy}"/>
|
|
394 <!-- deploy area -->
|
|
395 <copy file="${build.dir}/${final.name}.job"
|
|
396 todir="${runtime.deploy}"/>
|
|
397 <copy todir="${runtime.deploy}/bin">
|
|
398 <fileset dir="src/bin"/>
|
|
399 </copy>
|
|
400 <chmod perm="ugo+x" type="file">
|
|
401 <fileset dir="${runtime.deploy}/bin"/>
|
|
402 </chmod>
|
|
403 <!-- local area -->
|
|
404 <copy file="${build.dir}/${final.name}.jar"
|
|
405 todir="${runtime.local}/lib"/>
|
|
406 <copy todir="${runtime.local}/lib/native">
|
|
407 <fileset dir="lib/native"/>
|
|
408 </copy>
|
|
409 <copy todir="${runtime.local}/conf">
|
|
410 <fileset dir="${conf.dir}" excludes="*.template"/>
|
|
411 </copy>
|
|
412 <copy todir="${runtime.local}/bin">
|
|
413 <fileset dir="src/bin"/>
|
|
414 </copy>
|
|
415 <chmod perm="ugo+x" type="file">
|
|
416 <fileset dir="${runtime.local}/bin"/>
|
|
417 </chmod>
|
|
418 <copy todir="${runtime.local}/lib">
|
|
419 <fileset dir="${build.dir}/lib"/>
|
|
420 </copy>
|
|
421 <copy todir="${runtime.local}/plugins">
|
|
422 <fileset dir="${build.dir}/plugins"/>
|
|
423 </copy>
|
|
424 <copy todir="${runtime.local}/test">
|
|
425 <fileset dir="${build.dir}/test"/>
|
|
426 </copy>
|
|
427 </target>
|
|
428
|
|
429 <!-- ================================================================== -->
|
|
430 <!-- Compile test code -->
|
|
431 <!-- ================================================================== -->
|
|
432 <target name="compile-core-test" depends="init, compile-core, resolve-test" description="--> compile test code">
|
|
433 <javac
|
|
434 encoding="${build.encoding}"
|
|
435 srcdir="${test.src.dir}"
|
|
436 includes="org/apache/nutch/**/*.java org/commoncrawl/**/*.java"
|
|
437 destdir="${test.build.classes}"
|
|
438 debug="${javac.debug}"
|
|
439 optimize="${javac.optimize}"
|
|
440 target="${javac.version}"
|
|
441 source="${javac.version}"
|
|
442 deprecation="${javac.deprecation}">
|
|
443 <compilerarg value="-Xlint:-path"/>
|
|
444 <classpath refid="test.classpath"/>
|
|
445 </javac>
|
|
446 </target>
|
|
447
|
|
448 <!-- ================================================================== -->
|
|
449 <!-- Run Nutch proxy -->
|
|
450 <!-- ================================================================== -->
|
|
451
|
|
452 <target name="proxy" depends="compile-core-test, job" description="--> run nutch proxy">
|
|
453 <java classname="org.apache.nutch.tools.proxy.ProxyTestbed" fork="true">
|
|
454 <classpath refid="test.classpath"/>
|
|
455 <arg value="-fake"/>
|
|
456 <!--
|
|
457 <arg value="-delay"/>
|
|
458 <arg value="-200"/>
|
|
459 -->
|
|
460 <jvmarg line="-Djavax.xml.parsers.DocumentBuilderFactory=com.sun.org.apache.xerces.internal.jaxp.DocumentBuilderFactoryImpl"/>
|
|
461 </java>
|
|
462 </target>
|
|
463
|
|
464 <!-- ================================================================== -->
|
|
465 <!-- Run Nutch benchmarking analysis -->
|
|
466 <!-- ================================================================== -->
|
|
467
|
|
468 <target name="benchmark" description="--> run nutch benchmarking analysis">
|
|
469 <java classname="org.apache.nutch.tools.Benchmark" fork="true">
|
|
470 <classpath refid="test.classpath"/>
|
|
471 <jvmarg line="-Xmx512m -Djavax.xml.parsers.DocumentBuilderFactory=com.sun.org.apache.xerces.internal.jaxp.DocumentBuilderFactoryImpl"/>
|
|
472 <arg value="-maxPerHost"/>
|
|
473 <arg value="10"/>
|
|
474 <arg value="-seeds"/>
|
|
475 <arg value="1"/>
|
|
476 <arg value="-depth"/>
|
|
477 <arg value="5"/>
|
|
478 </java>
|
|
479 </target>
|
|
480
|
|
481 <!-- ================================================================== -->
|
|
482 <!-- Run unit tests -->
|
|
483 <!-- ================================================================== -->
|
|
484 <target name="test" depends="test-core, test-plugins" description="--> run JUnit tests"/>
|
|
485
|
|
486 <target name="test-core" depends="compile-core-test, job" description="--> run core JUnit tests only">
|
|
487
|
|
488 <delete dir="${test.build.data}"/>
|
|
489 <mkdir dir="${test.build.data}"/>
|
|
490 <!--
|
|
491 copy resources needed in junit tests
|
|
492 -->
|
|
493 <copy todir="${test.build.data}">
|
|
494 <fileset dir="src/testresources" includes="**/*"/>
|
|
495 </copy>
|
|
496
|
|
497 <copy file="${test.src.dir}/log4j.properties"
|
|
498 todir="${test.build.classes}"/>
|
|
499
|
|
500 <copy file="${test.src.dir}/crawl-tests.xml"
|
|
501 todir="${test.build.classes}"/>
|
|
502
|
|
503 <copy file="${test.src.dir}/domain-urlfilter.txt"
|
|
504 todir="${test.build.classes}"/>
|
|
505
|
|
506 <copy file="${test.src.dir}/filter-all.txt"
|
|
507 todir="${test.build.classes}"/>
|
|
508
|
|
509 <junit printsummary="yes" haltonfailure="no" fork="yes"
|
|
510 forkmode="once" dir="${basedir}"
|
|
511 errorProperty="tests.failed" failureProperty="tests.failed"
|
|
512 maxmemory="1000m">
|
|
513 <sysproperty key="test.build.data" value="${test.build.data}"/>
|
|
514 <sysproperty key="test.src.dir" value="${test.src.dir}"/>
|
|
515 <sysproperty key="javax.xml.parsers.DocumentBuilderFactory" value="com.sun.org.apache.xerces.internal.jaxp.DocumentBuilderFactoryImpl"/>
|
|
516 <classpath refid="test.classpath"/>
|
|
517 <formatter type="${test.junit.output.format}" />
|
|
518 <batchtest todir="${test.build.dir}" unless="testcase">
|
|
519 <fileset dir="${test.src.dir}"
|
|
520 includes="**/Test*.java" excludes="**/${test.exclude}.java" />
|
|
521 </batchtest>
|
|
522 <batchtest todir="${test.build.dir}" if="testcase">
|
|
523 <fileset dir="${test.src.dir}" includes="**/${testcase}.java"/>
|
|
524 </batchtest>
|
|
525 </junit>
|
|
526
|
|
527 <fail if="tests.failed">Tests failed!</fail>
|
|
528
|
|
529 </target>
|
|
530
|
|
531 <target name="test-plugins" depends="resolve-test, compile" description="--> run plugin JUnit tests only">
|
|
532 <ant dir="src/plugin" target="test" inheritAll="false"/>
|
|
533 </target>
|
|
534
|
|
535 <target name="test-plugin" depends="resolve-test, compile" description="--> run a single plugin's JUnit tests">
|
|
536 <ant dir="src/plugin" target="test-single" inheritAll="false"/>
|
|
537 </target>
|
|
538
|
|
539 <target name="nightly" depends="test, tar-src, zip-src" description="--> run the nightly target build">
|
|
540 </target>
|
|
541
|
|
542 <!-- ================================================================== -->
|
|
543 <!-- Ivy targets -->
|
|
544 <!-- ================================================================== -->
|
|
545
|
|
546 <!-- target: resolve ================================================= -->
|
|
547 <target name="resolve-default" depends="clean-default-lib, init" description="--> resolve and retrieve dependencies with ivy">
|
|
548 <ivy:resolve file="${ivy.file}" conf="default" log="download-only"/>
|
|
549 <ivy:retrieve pattern="${build.lib.dir}/[artifact]-[revision].[ext]" symlink="false" log="quiet"/>
|
|
550 <antcall target="copy-libs"/>
|
|
551 </target>
|
|
552
|
|
553 <target name="resolve-test" depends="clean-test-lib, init" description="--> resolve and retrieve dependencies with ivy">
|
|
554 <ivy:resolve file="${ivy.file}" conf="test" log="download-only"/>
|
|
555 <ivy:retrieve pattern="${test.build.lib.dir}/[artifact]-[revision].[ext]" symlink="false" log="quiet"/>
|
|
556 <antcall target="copy-libs"/>
|
|
557 </target>
|
|
558
|
|
559 <target name="copy-libs" description="--> copy the libs in lib, which are not ivy enabled">
|
|
560 <!-- copy the libs in lib, which are not ivy enabled-->
|
|
561 <copy todir="${build.lib.dir}/" failonerror="false">
|
|
562 <fileset dir="${lib.dir}" includes="**/*.jar"/>
|
|
563 </copy>
|
|
564 </target>
|
|
565
|
|
566 <!-- target: publish-local =========================================== -->
|
|
567 <target name="publish-local" depends="jar" description="--> publish this project in the local ivy repository">
|
|
568 <ivy:publish artifactspattern="${build.dir}/[artifact]-${version}.[ext]"
|
|
569 resolver="local"
|
|
570 pubrevision="${version}"
|
|
571 pubdate="${now}"
|
|
572 status="integration"
|
|
573 forcedeliver="true"
|
|
574 overwrite="true"
|
|
575 />
|
|
576 <echo message="project ${ant.project.name} published locally with version ${version}" />
|
|
577 </target>
|
|
578
|
|
579 <!-- target: report ================================================== -->
|
|
580 <target name="report" depends="resolve-test" description="--> generates a report of dependencies">
|
|
581 <ivy:report todir="${build.dir}" xml="true"/>
|
|
582 </target>
|
|
583
|
|
584 <!-- target: 3rd-party licenses report =============================== -->
|
|
585 <target name="report-licenses" depends="resolve-default" description="--> generates a report of licenses of dependencies">
|
|
586 <ivy:report todir="${build.dir}" xml="false" graph="false" xslfile="ivy/ivy-report-license.xsl"
|
|
587 outputpattern="[organisation]-[module]-[conf]-3rd-party-licenses.tsv"/>
|
|
588 </target>
|
|
589
|
|
590 <!-- target: ivy-init ================================================ -->
|
|
591 <target name="ivy-init" depends="ivy-probe-antlib, ivy-init-antlib" description="--> initialise Ivy settings">
|
|
592 <ivy:settings file="${ivy.dir}/ivysettings.xml" />
|
|
593 </target>
|
|
594
|
|
595 <!-- target: ivy-probe-antlib ======================================== -->
|
|
596 <target name="ivy-probe-antlib" description="--> probe the antlib library">
|
|
597 <condition property="ivy.found">
|
|
598 <typefound uri="antlib:org.apache.ivy.ant" name="cleancache" />
|
|
599 </condition>
|
|
600 </target>
|
|
601
|
|
602 <!-- target: ivy-download ============================================ -->
|
|
603 <target name="ivy-download" description="--> download ivy">
|
|
604 <available file="${ivy.jar}" property="ivy.jar.found"/>
|
|
605 <antcall target="ivy-download-unchecked"/>
|
|
606 </target>
|
|
607
|
|
608 <!-- target: ivy-download-unchecked ================================== -->
|
|
609 <target name="ivy-download-unchecked" unless="ivy.jar.found" description="--> fetch any ivy file">
|
|
610 <get src="${ivy.repo.url}" dest="${ivy.jar}" usetimestamp="true" />
|
|
611 </target>
|
|
612
|
|
613 <!-- target: ivy-init-antlib ========================================= -->
|
|
614 <target name="ivy-init-antlib" depends="ivy-download" unless="ivy.found" description="--> attempt to use Ivy with Antlib">
|
|
615 <typedef uri="antlib:org.apache.ivy.ant" onerror="fail" loaderRef="ivyLoader">
|
|
616 <classpath>
|
|
617 <pathelement location="${ivy.jar}" />
|
|
618 </classpath>
|
|
619 </typedef>
|
|
620 <fail>
|
|
621 <condition>
|
|
622 <not>
|
|
623 <typefound uri="antlib:org.apache.ivy.ant" name="cleancache" />
|
|
624 </not>
|
|
625 </condition>
|
|
626 You need Apache Ivy 2.0 or later from http://ant.apache.org/
|
|
627 It could not be loaded from ${ivy.repo.url}
|
|
628 </fail>
|
|
629 </target>
|
|
630
|
|
631 <!-- Check dependencies for security vulnerabilities -->
|
|
632 <!-- requires installation of OWASP dependency check tool, see -->
|
|
633 <!-- https://jeremylong.github.io/DependencyCheck/dependency-check-ant/index.html -->
|
|
634 <!-- get http://dl.bintray.com/jeremy-long/owasp/dependency-check-ant-3.3.2-release.zip -->
|
|
635 <!-- and unzip in directory ./ivy/ -->
|
|
636 <property name="dependency-check.home" value="${ivy.dir}/dependency-check-ant/"/>
|
|
637 <path id="dependency-check.path">
|
|
638 <pathelement location="${dependency-check.home}/dependency-check-ant.jar"/>
|
|
639 <fileset dir="${dependency-check.home}/lib" erroronmissingdir="false">
|
|
640 <include name="*.jar"/>
|
|
641 </fileset>
|
|
642 </path>
|
|
643 <taskdef resource="dependency-check-taskdefs.properties" onerror="ignore">
|
|
644 <classpath refid="dependency-check.path" />
|
|
645 </taskdef>
|
|
646 <target name="report-vulnerabilities" description="--> check dependencies for security vulnerabilities">
|
|
647 <dependency-check projectname="${name}"
|
|
648 reportoutputdirectory="${build.dir}"
|
|
649 reportformat="ALL">
|
|
650 <suppressionfile path="${dependency-check.home}/dependency-check-suppressions.xml" />
|
|
651 <retirejsFilter regex="copyright.*jeremy long" />
|
|
652 <fileset dir="${build.dir}">
|
|
653 <include name="lib/*.jar"/>
|
|
654 <include name="plugins/*/*.jar"/>
|
|
655 </fileset>
|
|
656 </dependency-check>
|
|
657 </target>
|
|
658
|
|
659 <!-- ================================================================== -->
|
|
660 <!-- Documentation -->
|
|
661 <!-- ================================================================== -->
|
|
662 <target name="javadoc" depends="compile" description="--> generate Javadoc">
|
|
663 <fail message="Unsupported Java version: ${java.version}. Javadoc requires Java version 7u25 or greater. See https://issues.apache.org/jira/browse/NUTCH-1590">
|
|
664 <condition>
|
|
665 <or>
|
|
666 <matches string="${java.version}" pattern="1.7.0_2[01234].+" casesensitive="false" />
|
|
667 <matches string="${java.version}" pattern="1.7.0_1.+" casesensitive="false" />
|
|
668 <equals arg1="${ant.java.version}" arg2="1.6" />
|
|
669 <equals arg1="${ant.java.version}" arg2="1.5" />
|
|
670 </or>
|
|
671 </condition>
|
|
672 </fail>
|
|
673 <mkdir dir="${build.javadoc}"/>
|
|
674 <javadoc
|
|
675 overview="${src.dir}/overview.html"
|
|
676 destdir="${build.javadoc}"
|
|
677 author="true"
|
|
678 version="true"
|
|
679 use="true"
|
|
680 windowtitle="${name} ${version} API"
|
|
681 doctitle="${name} ${version} API"
|
|
682 bottom="Copyright &copy; ${year} The Apache Software Foundation"
|
|
683 >
|
|
684 <arg value="${javadoc.proxy.host}"/>
|
|
685 <arg value="${javadoc.proxy.port}"/>
|
|
686
|
|
687 <packageset dir="${src.dir}"/>
|
|
688 <packageset dir="${plugins.dir}/any23/src/java/" />
|
|
689 <packageset dir="${plugins.dir}/creativecommons/src/java"/>
|
|
690 <packageset dir="${plugins.dir}/feed/src/java"/>
|
|
691 <packageset dir="${plugins.dir}/headings/src/java"/>
|
|
692 <packageset dir="${plugins.dir}/exchange-jexl/src/java"/>
|
|
693 <packageset dir="${plugins.dir}/index-anchor/src/java"/>
|
|
694 <packageset dir="${plugins.dir}/index-basic/src/java"/>
|
|
695 <packageset dir="${plugins.dir}/index-geoip/src/java"/>
|
|
696 <packageset dir="${plugins.dir}/index-jexl-filter/src/java"/>
|
|
697 <packageset dir="${plugins.dir}/index-links/src/java"/>
|
|
698 <packageset dir="${plugins.dir}/index-metadata/src/java"/>
|
|
699 <packageset dir="${plugins.dir}/index-more/src/java"/>
|
|
700 <packageset dir="${plugins.dir}/index-replace/src/java"/>
|
|
701 <packageset dir="${plugins.dir}/index-static/src/java"/>
|
|
702 <packageset dir="${plugins.dir}/indexer-cloudsearch/src/java/" />
|
|
703 <packageset dir="${plugins.dir}/indexer-csv/src/java"/>
|
|
704 <packageset dir="${plugins.dir}/indexer-dummy/src/java"/>
|
|
705 <packageset dir="${plugins.dir}/indexer-elastic-rest/src/java/"/>
|
|
706 <packageset dir="${plugins.dir}/indexer-elastic/src/java/" />
|
|
707 <packageset dir="${plugins.dir}/indexer-kafka/src/java/" />
|
|
708 <packageset dir="${plugins.dir}/indexer-rabbit/src/java"/>
|
|
709 <packageset dir="${plugins.dir}/indexer-solr/src/java"/>
|
|
710 <packageset dir="${plugins.dir}/language-identifier/src/java"/>
|
|
711 <packageset dir="${plugins.dir}/lib-htmlunit/src/java"/>
|
|
712 <packageset dir="${plugins.dir}/lib-http/src/java"/>
|
|
713 <packageset dir="${plugins.dir}/lib-rabbitmq/src/java"/>
|
|
714 <packageset dir="${plugins.dir}/lib-regex-filter/src/java"/>
|
|
715 <packageset dir="${plugins.dir}/lib-selenium/src/java"/>
|
|
716 <packageset dir="${plugins.dir}/microformats-reltag/src/java"/>
|
|
717 <packageset dir="${plugins.dir}/mimetype-filter/src/java"/>
|
|
718 <packageset dir="${plugins.dir}/parse-ext/src/java"/>
|
|
719 <packageset dir="${plugins.dir}/parse-html/src/java"/>
|
|
720 <packageset dir="${plugins.dir}/parse-js/src/java"/>
|
|
721 <packageset dir="${plugins.dir}/parse-metatags/src/java"/>
|
|
722 <packageset dir="${plugins.dir}/parse-swf/src/java"/>
|
|
723 <packageset dir="${plugins.dir}/parse-tika/src/java"/>
|
|
724 <packageset dir="${plugins.dir}/parse-zip/src/java"/>
|
|
725 <packageset dir="${plugins.dir}/parsefilter-naivebayes/src/java"/>
|
|
726 <packageset dir="${plugins.dir}/parsefilter-regex/src/java"/>
|
|
727 <packageset dir="${plugins.dir}/protocol-file/src/java"/>
|
|
728 <packageset dir="${plugins.dir}/protocol-ftp/src/java"/>
|
|
729 <packageset dir="${plugins.dir}/protocol-htmlunit/src/java"/>
|
|
730 <packageset dir="${plugins.dir}/protocol-http/src/java"/>
|
|
731 <packageset dir="${plugins.dir}/protocol-httpclient/src/java"/>
|
|
732 <packageset dir="${plugins.dir}/protocol-interactiveselenium/src/java"/>
|
|
733 <packageset dir="${plugins.dir}/protocol-okhttp/src/java"/>
|
|
734 <packageset dir="${plugins.dir}/protocol-selenium/src/java"/>
|
|
735 <packageset dir="${plugins.dir}/publish-rabbitmq/src/java"/>
|
|
736 <packageset dir="${plugins.dir}/scoring-adaptive/src/java"/>
|
|
737 <packageset dir="${plugins.dir}/scoring-depth/src/java"/>
|
|
738 <packageset dir="${plugins.dir}/scoring-link/src/java"/>
|
|
739 <packageset dir="${plugins.dir}/scoring-opic/src/java"/>
|
|
740 <packageset dir="${plugins.dir}/scoring-orphan/src/java"/>
|
|
741 <packageset dir="${plugins.dir}/scoring-similarity/src/java"/>
|
|
742 <packageset dir="${plugins.dir}/subcollection/src/java"/>
|
|
743 <packageset dir="${plugins.dir}/tld/src/java"/>
|
|
744 <packageset dir="${plugins.dir}/urlfilter-automaton/src/java"/>
|
|
745 <packageset dir="${plugins.dir}/urlfilter-domain/src/java"/>
|
|
746 <packageset dir="${plugins.dir}/urlfilter-domainblacklist/src/java"/>
|
|
747 <packageset dir="${plugins.dir}/urlfilter-fast/src/java"/>
|
|
748 <packageset dir="${plugins.dir}/urlfilter-ignoreexempt/src/java"/>
|
|
749 <packageset dir="${plugins.dir}/urlfilter-prefix/src/java"/>
|
|
750 <packageset dir="${plugins.dir}/urlfilter-regex/src/java"/>
|
|
751 <packageset dir="${plugins.dir}/urlfilter-suffix/src/java"/>
|
|
752 <packageset dir="${plugins.dir}/urlfilter-validator/src/java"/>
|
|
753 <packageset dir="${plugins.dir}/urlmeta/src/java"/>
|
|
754 <packageset dir="${plugins.dir}/urlnormalizer-ajax/src/java"/>
|
|
755 <packageset dir="${plugins.dir}/urlnormalizer-basic/src/java"/>
|
|
756 <packageset dir="${plugins.dir}/urlnormalizer-host/src/java"/>
|
|
757 <packageset dir="${plugins.dir}/urlnormalizer-pass/src/java"/>
|
|
758 <packageset dir="${plugins.dir}/urlnormalizer-protocol/src/java"/>
|
|
759 <packageset dir="${plugins.dir}/urlnormalizer-querystring/src/java"/>
|
|
760 <packageset dir="${plugins.dir}/urlnormalizer-regex/src/java"/>
|
|
761 <packageset dir="${plugins.dir}/urlnormalizer-slash/src/java"/>
|
|
762
|
|
763 <link href="${javadoc.link.java}"/>
|
|
764 <link href="${javadoc.link.hadoop}"/>
|
|
765 <link href="${javadoc.link.lucene.core}"/>
|
|
766 <link href="${javadoc.link.lucene.analyzers-common}"/>
|
|
767 <link href="${javadoc.link.solr-solrj}"/>
|
|
768
|
|
769 <classpath refid="classpath"/>
|
|
770 <classpath>
|
|
771 <fileset dir="${plugins.dir}" >
|
|
772 <include name="**/*.jar"/>
|
|
773 </fileset>
|
|
774 </classpath>
|
|
775
|
|
776 <group title="Core" packages="org.apache.nutch.*"/>
|
|
777 <group title="Plugins API" packages="${plugins.api}"/>
|
|
778 <group title="Protocol Plugins" packages="${plugins.protocol}"/>
|
|
779 <group title="URL Filter Plugins" packages="${plugins.urlfilter}"/>
|
|
780 <group title="URL Normalizer Plugins" packages="${plugins.urlnormalizer}"/>
|
|
781 <group title="Scoring Plugins" packages="${plugins.scoring}"/>
|
|
782 <group title="Parse Plugins" packages="${plugins.parse}"/>
|
|
783 <group title="Parse Filter Plugins" packages="${plugins.parsefilter}"/>
|
|
784 <group title="Publisher Plugins" packages="${plugins.publisher}"/>
|
|
785 <group title="Exchange Plugins" packages="${plugins.exchange}"/>
|
|
786 <group title="Indexing Filter Plugins" packages="${plugins.index}"/>
|
|
787 <group title="Indexer Plugins" packages="${plugins.indexer}"/>
|
|
788 <group title="Misc. Plugins" packages="${plugins.misc}"/>
|
|
789 </javadoc>
|
|
790 <!-- Copy the plugin.dtd file to the plugin doc-files dir -->
|
|
791 <copy file="${plugins.dir}/plugin.dtd"
|
|
792 todir="${build.javadoc}/org/apache/nutch/plugin/doc-files"/>
|
|
793 </target>
|
|
794
|
|
795 <target name="default-doc" description="--> generate default Nutch documentation">
|
|
796 <style basedir="${conf.dir}" destdir="${docs.dir}"
|
|
797 includes="nutch-default.xml" style="conf/nutch-conf.xsl"/>
|
|
798 </target>
|
|
799
|
|
800 <!-- ================================================================== -->
|
|
801 <!-- D I S T R I B U T I O N -->
|
|
802 <!-- ================================================================== -->
|
|
803 <!-- -->
|
|
804 <!-- ================================================================== -->
|
|
805 <target name="package-src" depends="runtime, javadoc" description="--> generate source distribution package">
|
|
806 <mkdir dir="${dist.dir}"/>
|
|
807 <mkdir dir="${src.dist.version.dir}"/>
|
|
808 <mkdir dir="${src.dist.version.dir}/lib"/>
|
|
809 <mkdir dir="${src.dist.version.dir}/docs"/>
|
|
810 <mkdir dir="${src.dist.version.dir}/docs/api"/>
|
|
811 <mkdir dir="${src.dist.version.dir}/ivy"/>
|
|
812
|
|
813 <copy todir="${src.dist.version.dir}/lib" includeEmptyDirs="false">
|
|
814 <fileset dir="lib"/>
|
|
815 </copy>
|
|
816
|
|
817 <copy todir="${src.dist.version.dir}/conf">
|
|
818 <fileset dir="${conf.dir}" excludes="**/*.template"/>
|
|
819 </copy>
|
|
820
|
|
821 <copy todir="${src.dist.version.dir}/docs/api">
|
|
822 <fileset dir="${build.javadoc}"/>
|
|
823 </copy>
|
|
824
|
|
825 <copy todir="${src.dist.version.dir}">
|
|
826 <fileset dir=".">
|
|
827 <include name="*.txt" />
|
|
828 <!--<include name="KEYS" />-->
|
|
829 </fileset>
|
|
830 </copy>
|
|
831
|
|
832 <copy todir="${src.dist.version.dir}/src" includeEmptyDirs="true">
|
|
833 <fileset dir="src"/>
|
|
834 </copy>
|
|
835
|
|
836 <copy todir="${src.dist.version.dir}/ivy" includeEmptyDirs="true">
|
|
837 <fileset dir="ivy"/>
|
|
838 </copy>
|
|
839
|
|
840 <copy todir="${src.dist.version.dir}/" file="build.xml"/>
|
|
841 <copy todir="${src.dist.version.dir}/" file="default.properties"/>
|
|
842
|
|
843 </target>
|
|
844
|
|
845 <target name="package-bin" depends="runtime, javadoc" description="--> generate binary distribution package">
|
|
846 <mkdir dir="${dist.dir}"/>
|
|
847 <mkdir dir="${bin.dist.version.dir}"/>
|
|
848 <mkdir dir="${bin.dist.version.dir}/lib"/>
|
|
849 <mkdir dir="${bin.dist.version.dir}/bin"/>
|
|
850 <mkdir dir="${bin.dist.version.dir}/conf"/>
|
|
851 <mkdir dir="${bin.dist.version.dir}/docs"/>
|
|
852 <mkdir dir="${bin.dist.version.dir}/docs/api"/>
|
|
853 <mkdir dir="${bin.dist.version.dir}/plugins"/>
|
|
854
|
|
855 <copy todir="${bin.dist.version.dir}/lib" includeEmptyDirs="false">
|
|
856 <fileset dir="runtime/local/lib"/>
|
|
857 </copy>
|
|
858
|
|
859 <copy todir="${bin.dist.version.dir}/bin">
|
|
860 <fileset dir="runtime/local/bin"/>
|
|
861 </copy>
|
|
862
|
|
863 <chmod perm="ugo+x" type="file">
|
|
864 <fileset dir="${bin.dist.version.dir}/bin"/>
|
|
865 </chmod>
|
|
866
|
|
867 <copy todir="${bin.dist.version.dir}/conf">
|
|
868 <fileset dir="runtime/local/conf" excludes="**/*.template"/>
|
|
869 </copy>
|
|
870
|
|
871 <copy todir="${bin.dist.version.dir}/docs/api">
|
|
872 <fileset dir="${build.javadoc}"/>
|
|
873 </copy>
|
|
874
|
|
875 <copy todir="${bin.dist.version.dir}">
|
|
876 <fileset dir=".">
|
|
877 <include name="*.txt" />
|
|
878 </fileset>
|
|
879 </copy>
|
|
880
|
|
881 <copy todir="${bin.dist.version.dir}/plugins" includeEmptyDirs="true">
|
|
882 <fileset dir="runtime/local/plugins"/>
|
|
883 </copy>
|
|
884
|
|
885 </target>
|
|
886
|
|
887 <!-- ================================================================== -->
|
|
888 <!-- Make src release tarball -->
|
|
889 <!-- ================================================================== -->
|
|
890 <target name="tar-src" depends="package-src" description="--> generate src.tar.gz distribution package">
|
|
891 <tar compression="gzip" longfile="gnu"
|
|
892 destfile="${src.dist.version.dir}.tar.gz">
|
|
893 <tarfileset dir="${src.dist.version.dir}" mode="664" prefix="${final.name}">
|
|
894 <exclude name="src/bin/*" />
|
|
895 <include name="**" />
|
|
896 </tarfileset>
|
|
897 <tarfileset dir="${src.dist.version.dir}" mode="755" prefix="${final.name}">
|
|
898 <include name="src/bin/*" />
|
|
899 </tarfileset>
|
|
900 </tar>
|
|
901 </target>
|
|
902
|
|
903 <!-- ================================================================== -->
|
|
904 <!-- Make bin release tarball -->
|
|
905 <!-- ================================================================== -->
|
|
906 <target name="tar-bin" depends="package-bin" description="--> generate bin.tar.gz distribution package">
|
|
907 <tar compression="gzip" longfile="gnu"
|
|
908 destfile="${bin.dist.version.dir}.tar.gz">
|
|
909 <tarfileset dir="${bin.dist.version.dir}" mode="664" prefix="${final.name}">
|
|
910 <exclude name="bin/*" />
|
|
911 <include name="**" />
|
|
912 </tarfileset>
|
|
913 <tarfileset dir="${bin.dist.version.dir}" mode="755" prefix="${final.name}">
|
|
914 <include name="bin/*" />
|
|
915 </tarfileset>
|
|
916 </tar>
|
|
917 </target>
|
|
918
|
|
919 <!-- ================================================================== -->
|
|
920 <!-- Make src release zip -->
|
|
921 <!-- ================================================================== -->
|
|
922 <target name="zip-src" depends="package-src" description="--> generate src.zip distribution package">
|
|
923 <zip compress="true" casesensitive="yes"
|
|
924 destfile="${src.dist.version.dir}.zip">
|
|
925 <zipfileset dir="${src.dist.version.dir}" filemode="664" prefix="${final.name}">
|
|
926 <exclude name="src/bin/*" />
|
|
927 <include name="**" />
|
|
928 </zipfileset>
|
|
929 <zipfileset dir="${src.dist.version.dir}" filemode="755" prefix="${final.name}">
|
|
930 <include name="src/bin/*" />
|
|
931 </zipfileset>
|
|
932 </zip>
|
|
933 </target>
|
|
934
|
|
935 <!-- ================================================================== -->
|
|
936 <!-- Make bin release zip -->
|
|
937 <!-- ================================================================== -->
|
|
938 <target name="zip-bin" depends="package-bin" description="--> generate bin.zip distribution package">
|
|
939 <zip compress="true" casesensitive="yes"
|
|
940 destfile="${bin.dist.version.dir}.zip">
|
|
941 <zipfileset dir="${bin.dist.version.dir}" filemode="664" prefix="${final.name}">
|
|
942 <exclude name="bin/*" />
|
|
943 <include name="**" />
|
|
944 </zipfileset>
|
|
945 <zipfileset dir="${bin.dist.version.dir}" filemode="755" prefix="${final.name}">
|
|
946 <include name="bin/*" />
|
|
947 </zipfileset>
|
|
948 </zip>
|
|
949 </target>
|
|
950
|
|
951 <!-- ================================================================== -->
|
|
952 <!-- Clean. Delete the build files, and their directories -->
|
|
953 <!-- ================================================================== -->
|
|
954
|
|
955 <!-- target: clean =================================================== -->
|
|
956 <target name="clean" depends="clean-build, clean-lib, clean-dist, clean-runtime" description="--> clean the project" />
|
|
957
|
|
958 <!-- target: clean-eclipse ============================================ -->
|
|
959 <target name="clean-eclipse" depends="clean-build, clean-lib, clean-dist" description="--> cleans the eclipse project"/>
|
|
960
|
|
961 <!-- target: clean-local ============================================= -->
|
|
962 <target name="clean-local" depends=""
|
|
963 description="--> cleans the local repository for the current module">
|
|
964 <delete dir="${ivy.local.default.root}/${ivy.organisation}/${ivy.module}"/>
|
|
965 </target>
|
|
966
|
|
967 <!-- target: clean-lib =============================================== -->
|
|
968 <target name="clean-lib" depends="clean-default-lib, clean-test-lib"
|
|
969 description="--> clean the project libraries directories (dependencies: default + test)">
|
|
970 </target>
|
|
971 <!-- target: clean-default-lib =============================================== -->
|
|
972 <target name="clean-default-lib" description="--> clean the project libraries directory (dependencies)">
|
|
973 <delete includeemptydirs="true" dir="${build.lib.dir}"/>
|
|
974 </target>
|
|
975 <!-- target: clean-test-lib =============================================== -->
|
|
976 <target name="clean-test-lib" description="--> clean the project test libraries directory (dependencies)">
|
|
977 <delete includeemptydirs="true" dir="${test.build.lib.dir}"/>
|
|
978 </target>
|
|
979
|
|
980 <!-- target: clean-build ============================================= -->
|
|
981 <target name="clean-build" description="--> clean the project built files">
|
|
982 <delete includeemptydirs="true" dir="${build.dir}"/>
|
|
983 </target>
|
|
984
|
|
985 <!-- target: clean-dist ============================================= -->
|
|
986 <target name="clean-dist" description="--> clean the project dist files">
|
|
987 <delete includeemptydirs="true" dir="${dist.dir}"/>
|
|
988 </target>
|
|
989
|
|
990 <!-- target: clean-cache ============================================= -->
|
|
991 <target name="clean-cache" depends=""
|
|
992 description="--> delete ivy cache">
|
|
993 <ivy:cleancache />
|
|
994 </target>
|
|
995
|
|
996 <target name="clean-runtime" description="--> clean the project runtime area">
|
|
997 <delete includeemptydirs="true" dir="${runtime.dir}"/>
|
|
998 </target>
|
|
999
|
|
1000 <!-- ================================================================== -->
|
|
1001 <!-- RAT targets -->
|
|
1002 <!-- ================================================================== -->
|
|
1003 <target name="rat-sources-typedef" description="--> run RAT antlib task">
|
|
1004 <typedef resource="org/apache/rat/anttasks/antlib.xml" >
|
|
1005 <classpath>
|
|
1006 <fileset dir="." includes="rat*.jar"/>
|
|
1007 </classpath>
|
|
1008 </typedef>
|
|
1009 </target>
|
|
1010
|
|
1011 <target name="rat-sources" depends="rat-sources-typedef"
|
|
1012 description="--> runs RAT tasks over src/java">
|
|
1013 <rat:report xmlns:rat="antlib:org.apache.rat.anttasks">
|
|
1014 <fileset dir="src">
|
|
1015 <include name="java/**/*"/>
|
|
1016 <include name="plugin/**/src/**/*"/>
|
|
1017 </fileset>
|
|
1018 </rat:report>
|
|
1019 </target>
|
|
1020
|
|
1021
|
|
1022 <!-- ================================================================== -->
|
|
1023 <!-- Eclipse targets -->
|
|
1024 <!-- ================================================================== -->
|
|
1025
|
|
1026 <!-- classpath for generating eclipse project -->
|
|
1027 <path id="eclipse.classpath">
|
|
1028 <fileset dir="${build.lib.dir}">
|
|
1029 <include name="*.jar" />
|
|
1030 <exclude name="ant-eclipse-1.0-jvm1.2.jar" />
|
|
1031 </fileset>
|
|
1032 <fileset dir="${build.plugins}">
|
|
1033 <include name="**/*.jar" />
|
|
1034 </fileset>
|
|
1035 <fileset dir="${test.build.lib.dir}">
|
|
1036 <include name="*.jar" />
|
|
1037 </fileset>
|
|
1038 </path>
|
|
1039
|
|
1040 <!-- target: ant-eclipse-download =================================== -->
|
|
1041 <target name="ant-eclipse-download" description="--> downloads the ant-eclipse binary.">
|
|
1042 <get src="https://downloads.sourceforge.net/project/ant-eclipse/ant-eclipse/1.0/ant-eclipse-1.0.bin.tar.bz2"
|
|
1043 dest="${build.dir}/ant-eclipse-1.0.bin.tar.bz2" usetimestamp="false" />
|
|
1044
|
|
1045 <untar src="${build.dir}/ant-eclipse-1.0.bin.tar.bz2"
|
|
1046 dest="${build.dir}" compression="bzip2">
|
|
1047 <patternset>
|
|
1048 <include name="lib/ant-eclipse-1.0-jvm1.2.jar"/>
|
|
1049 </patternset>
|
|
1050 </untar>
|
|
1051
|
|
1052 <delete file="${build.dir}/ant-eclipse-1.0.bin.tar.bz2" />
|
|
1053 </target>
|
|
1054
|
|
1055 <!-- target: eclipse ================================================ -->
|
|
1056 <target name="eclipse"
|
|
1057 depends="clean-eclipse,init,resolve-test,job,ant-eclipse-download"
|
|
1058 description="--> create eclipse project files">
|
|
1059
|
|
1060 <pathconvert property="eclipse.project">
|
|
1061 <path path="${basedir}"/>
|
|
1062 <regexpmapper from="^.*/([^/]+)$$" to="\1" handledirsep="yes"/>
|
|
1063 </pathconvert>
|
|
1064
|
|
1065 <taskdef name="eclipse"
|
|
1066 classname="prantl.ant.eclipse.EclipseTask"
|
|
1067 classpath="${build.dir}/lib/ant-eclipse-1.0-jvm1.2.jar" />
|
|
1068 <eclipse updatealways="true">
|
|
1069 <project name="${eclipse.project}" />
|
|
1070 <classpath>
|
|
1071 <library path="${conf.dir}" exported="false" />
|
|
1072 <library path="${basedir}/src/bin" exported="false" />
|
|
1073 <library pathref="eclipse.classpath" exported="false" />
|
|
1074
|
|
1075 <source path="${basedir}/src/java/" />
|
|
1076 <source path="${basedir}/src/test/" output="build/test/classes" />
|
|
1077
|
|
1078 <source path="${plugins.dir}/any23/src/java/" />
|
|
1079 <source path="${plugins.dir}/any23/src/test/" />
|
|
1080 <source path="${plugins.dir}/creativecommons/src/java/" />
|
|
1081 <source path="${plugins.dir}/creativecommons/src/test/" />
|
|
1082 <source path="${plugins.dir}/feed/src/java/" />
|
|
1083 <source path="${plugins.dir}/feed/src/test/" />
|
|
1084 <source path="${plugins.dir}/headings/src/java/" />
|
|
1085 <source path="${plugins.dir}/headings/src/test/" />
|
|
1086 <source path="${plugins.dir}/exchange-jexl/src/java/" />
|
|
1087 <source path="${plugins.dir}/index-anchor/src/java/" />
|
|
1088 <source path="${plugins.dir}/index-anchor/src/test/" />
|
|
1089 <source path="${plugins.dir}/index-basic/src/java/" />
|
|
1090 <source path="${plugins.dir}/index-basic/src/test/" />
|
|
1091 <source path="${plugins.dir}/index-geoip/src/java/" />
|
|
1092 <source path="${plugins.dir}/index-jexl-filter/src/java/" />
|
|
1093 <source path="${plugins.dir}/index-jexl-filter/src/test/" />
|
|
1094 <source path="${plugins.dir}/index-links/src/java/" />
|
|
1095 <source path="${plugins.dir}/index-links/src/test/" />
|
|
1096 <source path="${plugins.dir}/index-metadata/src/java/" />
|
|
1097 <source path="${plugins.dir}/index-more/src/java/" />
|
|
1098 <source path="${plugins.dir}/index-more/src/test/" />
|
|
1099 <source path="${plugins.dir}/index-replace/src/java/" />
|
|
1100 <source path="${plugins.dir}/index-replace/src/test/" />
|
|
1101 <source path="${plugins.dir}/index-static/src/java/" />
|
|
1102 <source path="${plugins.dir}/index-static/src/test/" />
|
|
1103 <source path="${plugins.dir}/indexer-cloudsearch/src/java/" />
|
|
1104 <source path="${plugins.dir}/indexer-csv/src/java"/>
|
|
1105 <source path="${plugins.dir}/indexer-csv/src/test"/>
|
|
1106 <source path="${plugins.dir}/indexer-dummy/src/java/" />
|
|
1107 <source path="${plugins.dir}/indexer-elastic-rest/src/java/"/>
|
|
1108 <source path="${plugins.dir}/indexer-elastic/src/java/" />
|
|
1109 <source path="${plugins.dir}/indexer-kafka/src/java/" />
|
|
1110 <source path="${plugins.dir}/indexer-rabbit/src/java/" />
|
|
1111 <source path="${plugins.dir}/indexer-solr/src/java/" />
|
|
1112 <source path="${plugins.dir}/language-identifier/src/java/" />
|
|
1113 <source path="${plugins.dir}/language-identifier/src/test/" />
|
|
1114 <source path="${plugins.dir}/lib-htmlunit/src/java/" />
|
|
1115 <source path="${plugins.dir}/lib-http/src/java/" />
|
|
1116 <source path="${plugins.dir}/lib-http/src/test/" />
|
|
1117 <source path="${plugins.dir}/lib-rabbitmq/src/java/" />
|
|
1118 <source path="${plugins.dir}/lib-regex-filter/src/java/" />
|
|
1119 <source path="${plugins.dir}/lib-regex-filter/src/test/" />
|
|
1120 <source path="${plugins.dir}/lib-selenium/src/java/" />
|
|
1121 <source path="${plugins.dir}/microformats-reltag/src/java/" />
|
|
1122 <source path="${plugins.dir}/mimetype-filter/src/java/" />
|
|
1123 <source path="${plugins.dir}/mimetype-filter/src/test/" />
|
|
1124 <source path="${plugins.dir}/parse-ext/src/java/" />
|
|
1125 <source path="${plugins.dir}/parse-ext/src/test/" />
|
|
1126 <source path="${plugins.dir}/parse-html/src/java/" />
|
|
1127 <source path="${plugins.dir}/parse-html/src/test/" />
|
|
1128 <source path="${plugins.dir}/parse-js/src/java/" />
|
|
1129 <source path="${plugins.dir}/parse-js/src/test/" />
|
|
1130 <source path="${plugins.dir}/parse-metatags/src/java/" />
|
|
1131 <source path="${plugins.dir}/parse-metatags/src/test/" />
|
|
1132 <source path="${plugins.dir}/parse-swf/src/java/" />
|
|
1133 <source path="${plugins.dir}/parse-swf/src/test/" />
|
|
1134 <source path="${plugins.dir}/parse-tika/src/java/" />
|
|
1135 <source path="${plugins.dir}/parse-tika/src/test/" />
|
|
1136 <source path="${plugins.dir}/parse-zip/src/java/" />
|
|
1137 <source path="${plugins.dir}/parse-zip/src/test/" />
|
|
1138 <source path="${plugins.dir}/parsefilter-naivebayes/src/java/" />
|
|
1139 <source path="${plugins.dir}/parsefilter-regex/src/java/" />
|
|
1140 <source path="${plugins.dir}/parsefilter-regex/src/test/" />
|
|
1141 <source path="${plugins.dir}/protocol-file/src/java/" />
|
|
1142 <source path="${plugins.dir}/protocol-file/src/test/" />
|
|
1143 <source path="${plugins.dir}/protocol-ftp/src/java/" />
|
|
1144 <source path="${plugins.dir}/protocol-htmlunit/src/java/" />
|
|
1145 <source path="${plugins.dir}/protocol-http/src/java/" />
|
|
1146 <source path="${plugins.dir}/protocol-http/src/test/" />
|
|
1147 <source path="${plugins.dir}/protocol-httpclient/src/java/" />
|
|
1148 <source path="${plugins.dir}/protocol-httpclient/src/test/" />
|
|
1149 <source path="${plugins.dir}/protocol-interactiveselenium/src/java/" />
|
|
1150 <source path="${plugins.dir}/protocol-okhttp/src/java/" />
|
|
1151 <source path="${plugins.dir}/protocol-okhttp/src/test/" />
|
|
1152 <source path="${plugins.dir}/protocol-selenium/src/java"/>
|
|
1153 <source path="${plugins.dir}/publish-rabbitmq/src/java"/>
|
|
1154 <source path="${plugins.dir}/scoring-adaptive/src/java"/>
|
|
1155 <source path="${plugins.dir}/scoring-depth/src/java/" />
|
|
1156 <source path="${plugins.dir}/scoring-link/src/java/" />
|
|
1157 <source path="${plugins.dir}/scoring-opic/src/java/" />
|
|
1158 <source path="${plugins.dir}/scoring-orphan/src/java"/>
|
|
1159 <source path="${plugins.dir}/scoring-orphan/src/test"/>
|
|
1160 <source path="${plugins.dir}/scoring-similarity/src/java/" />
|
|
1161 <source path="${plugins.dir}/subcollection/src/java/" />
|
|
1162 <source path="${plugins.dir}/subcollection/src/test/" />
|
|
1163 <source path="${plugins.dir}/tld/src/java/" />
|
|
1164 <source path="${plugins.dir}/urlfilter-automaton/src/java/" />
|
|
1165 <source path="${plugins.dir}/urlfilter-automaton/src/test/" />
|
|
1166 <source path="${plugins.dir}/urlfilter-domain/src/java/" />
|
|
1167 <source path="${plugins.dir}/urlfilter-domain/src/test/" />
|
|
1168 <source path="${plugins.dir}/urlfilter-domainblacklist/src/java/" />
|
|
1169 <source path="${plugins.dir}/urlfilter-domainblacklist/src/test/" />
|
|
1170 <source path="${plugins.dir}/urlfilter-fast/src/java/"/>
|
|
1171 <source path="${plugins.dir}/urlfilter-fast/src/test/"/>
|
|
1172 <source path="${plugins.dir}/urlfilter-ignoreexempt/src/java/" />
|
|
1173 <source path="${plugins.dir}/urlfilter-prefix/src/java/" />
|
|
1174 <source path="${plugins.dir}/urlfilter-prefix/src/test/" />
|
|
1175 <source path="${plugins.dir}/urlfilter-regex/src/java/" />
|
|
1176 <source path="${plugins.dir}/urlfilter-regex/src/test/" />
|
|
1177 <source path="${plugins.dir}/urlfilter-suffix/src/java/" />
|
|
1178 <source path="${plugins.dir}/urlfilter-suffix/src/test/" />
|
|
1179 <source path="${plugins.dir}/urlfilter-validator/src/java/" />
|
|
1180 <source path="${plugins.dir}/urlfilter-validator/src/test/" />
|
|
1181 <source path="${plugins.dir}/urlmeta/src/java/" />
|
|
1182 <source path="${plugins.dir}/urlnormalizer-ajax/src/java/" />
|
|
1183 <source path="${plugins.dir}/urlnormalizer-ajax/src/test/" />
|
|
1184 <source path="${plugins.dir}/urlnormalizer-basic/src/java/" />
|
|
1185 <source path="${plugins.dir}/urlnormalizer-basic/src/test/" />
|
|
1186 <source path="${plugins.dir}/urlnormalizer-host/src/java/" />
|
|
1187 <source path="${plugins.dir}/urlnormalizer-host/src/test/" />
|
|
1188 <source path="${plugins.dir}/urlnormalizer-pass/src/java/" />
|
|
1189 <source path="${plugins.dir}/urlnormalizer-pass/src/test/" />
|
|
1190 <source path="${plugins.dir}/urlnormalizer-protocol/src/java/" />
|
|
1191 <source path="${plugins.dir}/urlnormalizer-protocol/src/test/" />
|
|
1192 <source path="${plugins.dir}/urlnormalizer-querystring/src/java/" />
|
|
1193 <source path="${plugins.dir}/urlnormalizer-querystring/src/test/" />
|
|
1194 <source path="${plugins.dir}/urlnormalizer-regex/src/java/" />
|
|
1195 <source path="${plugins.dir}/urlnormalizer-regex/src/test/" />
|
|
1196 <source path="${plugins.dir}/urlnormalizer-slash/src/java/" />
|
|
1197 <source path="${plugins.dir}/urlnormalizer-slash/src/test/" />
|
|
1198
|
|
1199 <output path="${build.classes}" />
|
|
1200 </classpath>
|
|
1201 </eclipse>
|
|
1202 </target>
|
|
1203
|
|
1204 </project>
|