Mercurial > hg > cc > cirrus_home

comparison src/nutch-cc/build.xml @ 191:b5904d0bdfd4

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
orig, more or less
author Henry S. Thompson <ht@inf.ed.ac.uk>
date Wed, 25 Sep 2024 09:49:12 +0100
parents
children 4275eb6484da
comparison
equal deleted inserted replaced
190:f2bf736c2d40 191:b5904d0bdfd4
1 <?xml version="1.0"?>
2 <!--
3 Licensed to the Apache Software Foundation (ASF) under one or more
4 contributor license agreements. See the NOTICE file distributed with
5 this work for additional information regarding copyright ownership.
6 The ASF licenses this file to You under the Apache License, Version 2.0
7 (the "License"); you may not use this file except in compliance with
8 the License. You may obtain a copy of the License at
9
10 http://www.apache.org/licenses/LICENSE-2.0
11
12 Unless required by applicable law or agreed to in writing, software
13 distributed under the License is distributed on an "AS IS" BASIS,
14 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15 See the License for the specific language governing permissions and
16 limitations under the License.
17 -->
18 <project name="${name}" default="runtime" xmlns:ivy="antlib:org.apache.ivy.ant" xmlns:artifact="antlib:org.apache.maven.artifact.ant">
19
20 <!-- Load all the default properties, and any the user wants -->
21 <!-- to contribute (without having to type -D or edit this file -->
22 <property file="${user.home}/build.properties" />
23 <property file="${basedir}/build.properties" />
24 <property file="${basedir}/default.properties" />
25 <property name="release.dir" value="${build.dir}/release"/>
26
27 <!-- define Maven coordinates, repository url and artifacts name etc -->
28 <property name="groupId" value="org.apache.nutch" />
29 <property name="artifactId" value="nutch" />
30 <property name="maven-repository-url" value="https://repository.apache.org/service/local/staging/deploy/maven2" />
31 <property name="maven-repository-id" value="apache.releases.https" />
32 <property name="maven-jar" value="${release.dir}/${artifactId}-${version}.jar" />
33 <property name="maven-javadoc-jar" value="${release.dir}/${artifactId}-${version}-javadoc.jar" />
34 <property name="maven-sources-jar" value="${release.dir}/${artifactId}-${version}-sources.jar" />
35
36
37 <!-- the normal classpath -->
38 <path id="classpath">
39 <pathelement location="${build.classes}"/>
40 <fileset dir="${build.lib.dir}">
41 <include name="*.jar" />
42 </fileset>
43 </path>
44
45 <dirname property="plugins.classpath.dir" file="${build.plugins}"/>
46
47 <!-- the unit test classpath -->
48 <path id="test.classpath">
49 <pathelement location="${test.build.classes}" />
50 <pathelement location="${conf.dir}"/>
51 <pathelement location="${test.src.dir}"/>
52 <pathelement location="${plugins.classpath.dir}"/>
53 <path refid="classpath"/>
54 <pathelement location="${build.dir}/${final.name}.job" />
55 <fileset dir="${build.lib.dir}">
56 <include name="*.jar" />
57 </fileset>
58 <fileset dir="${test.build.lib.dir}">
59 <include name="*.jar" />
60 </fileset>
61 </path>
62
63 <presetdef name="javac">
64 <javac includeantruntime="false" />
65 </presetdef>
66
67 <!-- ====================================================== -->
68 <!-- Stuff needed by all targets -->
69 <!-- ====================================================== -->
70 <target name="init" depends="ivy-init" description="--> stuff required by all targets">
71 <mkdir dir="${build.dir}"/>
72 <mkdir dir="${build.classes}"/>
73 <mkdir dir="${release.dir}"/>
74
75 <mkdir dir="${test.build.dir}"/>
76 <mkdir dir="${test.build.classes}"/>
77 <mkdir dir="${test.build.lib.dir}"/>
78
79 <touch datetime="01/25/1971 2:00 pm">
80 <fileset dir="${conf.dir}" includes="**/*.template"/>
81 </touch>
82
83 <copy todir="${conf.dir}" verbose="true">
84 <fileset dir="${conf.dir}" includes="**/*.template"/>
85 <mapper type="glob" from="*.template" to="*"/>
86 </copy>
87 </target>
88
89 <!-- ====================================================== -->
90 <!-- Compile the Java files -->
91 <!-- ====================================================== -->
92 <target name="compile" depends="compile-core, compile-plugins" description="--> compile all Java files"/>
93
94 <target name="compile-core" depends="init, resolve-default" description="--> compile core Java files only">
95 <javac
96 encoding="${build.encoding}"
97 srcdir="${src.dir}"
98 includes="org/apache/nutch/**/*.java org/commoncrawl/**/*.java"
99 destdir="${build.classes}"
100 debug="${javac.debug}"
101 optimize="${javac.optimize}"
102 target="${javac.version}"
103 source="${javac.version}"
104 deprecation="${javac.deprecation}">
105 <compilerarg value="-Xlint:-path"/>
106 <classpath refid="classpath"/>
107 </javac>
108 <copy todir="${build.classes}">
109 <fileset dir="${src.dir}" includes="**/*.html" />
110 <fileset dir="${src.dir}" includes="**/*.css" />
111 <fileset dir="${src.dir}" includes="**/*.properties" />
112 </copy>
113 </target>
114
115 <target name="compile-plugins" depends="init, resolve-default" description="--> compile plugins only">
116 <ant dir="src/plugin" target="deploy" inheritAll="false"/>
117 </target>
118
119 <!-- ================================================================== -->
120 <!-- Make nutch.jar -->
121 <!-- ================================================================== -->
122 <!-- -->
123 <!-- ================================================================== -->
124 <target name="jar" depends="compile-core" description="--> make nutch.jar">
125 <copy file="${conf.dir}/nutch-default.xml"
126 todir="${build.classes}"/>
127 <copy file="${conf.dir}/nutch-site.xml"
128 todir="${build.classes}"/>
129 <jar jarfile="${build.dir}/${final.name}.jar"
130 basedir="${build.classes}">
131 <manifest>
132 </manifest>
133 </jar>
134 </target>
135
136 <!-- ================================================================== -->
137 <!-- Make Maven Central Release -->
138 <!-- ================================================================== -->
139 <!-- -->
140 <!-- ================================================================== -->
141 <target name="release" depends="compile-core" description="--> generate the release distribution">
142 <copy file="${conf.dir}/nutch-default.xml"
143 todir="${build.classes}"/>
144 <copy file="${conf.dir}/nutch-site.xml"
145 todir="${build.classes}"/>
146
147 <!-- build the main artifact -->
148 <jar jarfile="${maven-jar}" basedir="${build.classes}" />
149
150 <fail message="Unsupported Java version: ${java.version}. Javadoc requires Java version 7u25 or greater. See https://issues.apache.org/jira/browse/NUTCH-1590">
151 <condition>
152 <or>
153 <matches string="${java.version}" pattern="1.7.0_2[01234].+" casesensitive="false" />
154 <matches string="${java.version}" pattern="1.7.0_1.+" casesensitive="false" />
155 <equals arg1="${ant.java.version}" arg2="1.6" />
156 <equals arg1="${ant.java.version}" arg2="1.5" />
157 </or>
158 </condition>
159 </fail>
160
161 <!-- build the javadoc artifact -->
162 <javadoc
163 destdir="${release.dir}/javadoc"
164 overview="${src.dir}/overview.html"
165 author="true"
166 version="true"
167 use="true"
168 windowtitle="${name} ${version} API"
169 doctitle="${name} ${version} API"
170 bottom="Copyright &amp;copy; ${year} The Apache Software Foundation"
171 >
172 <arg value="${javadoc.proxy.host}"/>
173 <arg value="${javadoc.proxy.port}"/>
174
175 <packageset dir="${src.dir}"/>
176 <packageset dir="${plugins.dir}/any23/src/java/" />
177 <packageset dir="${plugins.dir}/creativecommons/src/java"/>
178 <packageset dir="${plugins.dir}/feed/src/java"/>
179 <packageset dir="${plugins.dir}/headings/src/java"/>
180 <packageset dir="${plugins.dir}/exchange-jexl/src/java"/>
181 <packageset dir="${plugins.dir}/index-anchor/src/java"/>
182 <packageset dir="${plugins.dir}/index-basic/src/java"/>
183 <packageset dir="${plugins.dir}/index-geoip/src/java"/>
184 <packageset dir="${plugins.dir}/index-jexl-filter/src/java"/>
185 <packageset dir="${plugins.dir}/index-links/src/java"/>
186 <packageset dir="${plugins.dir}/index-metadata/src/java"/>
187 <packageset dir="${plugins.dir}/index-more/src/java"/>
188 <packageset dir="${plugins.dir}/index-replace/src/java"/>
189 <packageset dir="${plugins.dir}/index-static/src/java"/>
190 <packageset dir="${plugins.dir}/indexer-cloudsearch/src/java/" />
191 <packageset dir="${plugins.dir}/indexer-csv/src/java"/>
192 <packageset dir="${plugins.dir}/indexer-dummy/src/java"/>
193 <packageset dir="${plugins.dir}/indexer-elastic-rest/src/java/"/>
194 <packageset dir="${plugins.dir}/indexer-elastic/src/java/" />
195 <packageset dir="${plugins.dir}/indexer-kafka/src/java/" />
196 <packageset dir="${plugins.dir}/indexer-rabbit/src/java"/>
197 <packageset dir="${plugins.dir}/indexer-solr/src/java"/>
198 <packageset dir="${plugins.dir}/language-identifier/src/java"/>
199 <packageset dir="${plugins.dir}/lib-htmlunit/src/java"/>
200 <packageset dir="${plugins.dir}/lib-http/src/java"/>
201 <packageset dir="${plugins.dir}/lib-rabbitmq/src/java"/>
202 <packageset dir="${plugins.dir}/lib-regex-filter/src/java"/>
203 <packageset dir="${plugins.dir}/lib-selenium/src/java"/>
204 <packageset dir="${plugins.dir}/microformats-reltag/src/java"/>
205 <packageset dir="${plugins.dir}/mimetype-filter/src/java"/>
206 <packageset dir="${plugins.dir}/parse-ext/src/java"/>
207 <packageset dir="${plugins.dir}/parse-html/src/java"/>
208 <packageset dir="${plugins.dir}/parse-js/src/java"/>
209 <packageset dir="${plugins.dir}/parse-metatags/src/java"/>
210 <packageset dir="${plugins.dir}/parse-swf/src/java"/>
211 <packageset dir="${plugins.dir}/parse-tika/src/java"/>
212 <packageset dir="${plugins.dir}/parse-zip/src/java"/>
213 <packageset dir="${plugins.dir}/parsefilter-naivebayes/src/java"/>
214 <packageset dir="${plugins.dir}/parsefilter-regex/src/java"/>
215 <packageset dir="${plugins.dir}/protocol-file/src/java"/>
216 <packageset dir="${plugins.dir}/protocol-ftp/src/java"/>
217 <packageset dir="${plugins.dir}/protocol-htmlunit/src/java"/>
218 <packageset dir="${plugins.dir}/protocol-http/src/java"/>
219 <packageset dir="${plugins.dir}/protocol-httpclient/src/java"/>
220 <packageset dir="${plugins.dir}/protocol-interactiveselenium/src/java"/>
221 <packageset dir="${plugins.dir}/protocol-okhttp/src/java"/>
222 <packageset dir="${plugins.dir}/protocol-selenium/src/java"/>
223 <packageset dir="${plugins.dir}/publish-rabbitmq/src/java"/>
224 <packageset dir="${plugins.dir}/scoring-adaptive/src/java"/>
225 <packageset dir="${plugins.dir}/scoring-depth/src/java"/>
226 <packageset dir="${plugins.dir}/scoring-link/src/java"/>
227 <packageset dir="${plugins.dir}/scoring-opic/src/java"/>
228 <packageset dir="${plugins.dir}/scoring-orphan/src/java"/>
229 <packageset dir="${plugins.dir}/scoring-similarity/src/java"/>
230 <packageset dir="${plugins.dir}/subcollection/src/java"/>
231 <packageset dir="${plugins.dir}/tld/src/java"/>
232 <packageset dir="${plugins.dir}/urlfilter-automaton/src/java"/>
233 <packageset dir="${plugins.dir}/urlfilter-domain/src/java"/>
234 <packageset dir="${plugins.dir}/urlfilter-domainblacklist/src/java"/>
235 <packageset dir="${plugins.dir}/urlfilter-fast/src/java"/>
236 <packageset dir="${plugins.dir}/urlfilter-ignoreexempt/src/java"/>
237 <packageset dir="${plugins.dir}/urlfilter-prefix/src/java"/>
238 <packageset dir="${plugins.dir}/urlfilter-regex/src/java"/>
239 <packageset dir="${plugins.dir}/urlfilter-suffix/src/java"/>
240 <packageset dir="${plugins.dir}/urlfilter-validator/src/java"/>
241 <packageset dir="${plugins.dir}/urlmeta/src/java"/>
242 <packageset dir="${plugins.dir}/urlnormalizer-ajax/src/java"/>
243 <packageset dir="${plugins.dir}/urlnormalizer-basic/src/java"/>
244 <packageset dir="${plugins.dir}/urlnormalizer-host/src/java"/>
245 <packageset dir="${plugins.dir}/urlnormalizer-pass/src/java"/>
246 <packageset dir="${plugins.dir}/urlnormalizer-protocol/src/java"/>
247 <packageset dir="${plugins.dir}/urlnormalizer-querystring/src/java"/>
248 <packageset dir="${plugins.dir}/urlnormalizer-regex/src/java"/>
249 <packageset dir="${plugins.dir}/urlnormalizer-slash/src/java"/>
250
251 <link href="${javadoc.link.java}"/>
252 <link href="${javadoc.link.hadoop}"/>
253 <link href="${javadoc.link.lucene.core}"/>
254 <link href="${javadoc.link.lucene.analyzers-common}"/>
255 <link href="${javadoc.link.solr-solrj}"/>
256
257 <classpath refid="classpath"/>
258 <classpath>
259 <fileset dir="${plugins.dir}" >
260 <include name="**/*.jar"/>
261 </fileset>
262 </classpath>
263
264 <group title="Core" packages="org.apache.nutch.*"/>
265 <group title="Plugins API" packages="${plugins.api}"/>
266 <group title="Protocol Plugins" packages="${plugins.protocol}"/>
267 <group title="URL Filter Plugins" packages="${plugins.urlfilter}"/>
268 <group title="URL Normalizer Plugins" packages="${plugins.urlnormalizer}"/>
269 <group title="Scoring Plugins" packages="${plugins.scoring}"/>
270 <group title="Parse Plugins" packages="${plugins.parse}"/>
271 <group title="Parse Filter Plugins" packages="${plugins.parsefilter}"/>
272 <group title="Publisher Plugins" packages="${plugins.publisher}"/>
273 <group title="Exchange Plugins" packages="${plugins.exchange}"/>
274 <group title="Indexing Filter Plugins" packages="${plugins.index}"/>
275 <group title="Indexer Plugins" packages="${plugins.indexer}"/>
276 <group title="Misc. Plugins" packages="${plugins.misc}"/>
277 </javadoc>
278 <jar jarfile="${maven-javadoc-jar}">
279 <fileset dir="${release.dir}/javadoc" />
280 </jar>
281
282 <!-- build the sources artifact -->
283 <jar jarfile="${maven-sources-jar}">
284 <fileset dir="${src.dir}" />
285 </jar>
286 </target>
287
288 <!-- ================================================================== -->
289 <!-- Deploy to Apache Nexus -->
290 <!-- ================================================================== -->
291 <!-- -->
292 <!-- ================================================================== -->
293 <target name="deploy" depends="release" description="--> deploy to Apache Nexus">
294
295 <!-- generate a pom file -->
296 <ivy:makepom ivyfile="${ivy.file}" pomfile="${basedir}/pom.xml" templatefile="ivy/mvn.template">
297 <mapping conf="default" scope="compile"/>
298 <mapping conf="runtime" scope="runtime"/>
299 </ivy:makepom>
300
301 <!-- sign and deploy the main artifact -->
302 <artifact:mvn>
303 <arg value="org.apache.maven.plugins:maven-gpg-plugin:1.5:sign-and-deploy-file" />
304 <arg value="-Durl=${maven-repository-url}" />
305 <arg value="-DrepositoryId=${maven-repository-id}" />
306 <arg value="-DpomFile=pom.xml" />
307 <arg value="-Dfile=${maven-jar}" />
308 <arg value="-Papache-release" />
309 </artifact:mvn>
310
311 <!-- sign and deploy the sources artifact -->
312 <artifact:mvn>
313 <arg value="org.apache.maven.plugins:maven-gpg-plugin:1.5:sign-and-deploy-file" />
314 <arg value="-Durl=${maven-repository-url}" />
315 <arg value="-DrepositoryId=${maven-repository-id}" />
316 <arg value="-DpomFile=pom.xml" />
317 <arg value="-Dfile=${maven-sources-jar}" />
318 <arg value="-Dclassifier=sources" />
319 <arg value="-Papache-release" />
320 </artifact:mvn>
321
322 <!-- sign and deploy the javadoc artifact -->
323 <artifact:mvn>
324 <arg value="org.apache.maven.plugins:maven-gpg-plugin:1.5:sign-and-deploy-file" />
325 <arg value="-Durl=${maven-repository-url}" />
326 <arg value="-DrepositoryId=${maven-repository-id}" />
327 <arg value="-DpomFile=pom.xml" />
328 <arg value="-Dfile=${maven-javadoc-jar}" />
329 <arg value="-Dclassifier=javadoc" />
330 <arg value="-Papache-release" />
331 </artifact:mvn>
332 </target>
333
334 <!-- ================================================================== -->
335 <!-- Generate REST API Documentation with Miredot -->
336 <!-- ================================================================== -->
337 <target name="restdocs" description="--> generate REST API Documentation with Miredot">
338
339 <!-- generate a pom file -->
340 <ivy:makepom ivyfile="${ivy.file}" pomfile="${basedir}/pom.xml" templatefile="ivy/mvn.template">
341 <mapping conf="default" scope="compile"/>
342 <mapping conf="runtime" scope="runtime"/>
343 </ivy:makepom>
344
345 <!--artifact:dependencies pathId="dependency.classpath">
346 <dependency groupId="log4j" artifactId="log4j" version="1.2.15" >
347 <exclusion groupId="javax.jms" artifactId="jms" />
348 <exclusion groupId="com.sun.jdmk" artifactId="jmxtools" />
349 <exclusion groupId="com.sun.jmx" artifactId="jmxri" />
350 </dependency>
351 </artifact:dependencies-->
352
353 <artifact:mvn>
354 <arg value="test"/>
355 <arg value="-e"/>
356 <arg value="-o"/>
357 <!-- run offline (-o): must not download dependencies as this is
358 done from http://repo1.maven.org/ hardwired in
359 maven-ant-tasks-2.1.3.jar, see NUTCH-2722.
360
361 Dependencies and plugins need to be resolved and cached locally beforehand
362 by running
363 `mvn dependency:resolve`
364 resp.
365 `mvn dependency:resolve-plugins`
366 after the pom.xml has been generated. -->
367 </artifact:mvn>
368 </target>
369
370 <!-- ================================================================== -->
371 <!-- Make job jar -->
372 <!-- ================================================================== -->
373 <!-- -->
374 <!-- ================================================================== -->
375 <target name="job" depends="compile" description="--> make nutch.job jar">
376 <jar jarfile="${build.dir}/${final.name}.job">
377 <!-- If the build.classes has the nutch config files because the jar
378 command command has run, exclude them. The conf directory has
379 them.
380 -->
381 <zipfileset dir="${build.classes}"
382 excludes="nutch-default.xml,nutch-site.xml"/>
383 <zipfileset dir="${conf.dir}" excludes="*.template,hadoop*.*"/>
384 <zipfileset dir="${build.lib.dir}" prefix="lib"
385 includes="**/*.jar" excludes="hadoop-*.jar,slf4j*.jar,log4j*.jar"/>
386 <zipfileset dir="${build.plugins}" prefix="classes/plugins"/>
387 </jar>
388 </target>
389
390 <target name="runtime" depends="jar, job" description="--> default target for running Nutch">
391 <mkdir dir="${runtime.dir}"/>
392 <mkdir dir="${runtime.local}"/>
393 <mkdir dir="${runtime.deploy}"/>
394 <!-- deploy area -->
395 <copy file="${build.dir}/${final.name}.job"
396 todir="${runtime.deploy}"/>
397 <copy todir="${runtime.deploy}/bin">
398 <fileset dir="src/bin"/>
399 </copy>
400 <chmod perm="ugo+x" type="file">
401 <fileset dir="${runtime.deploy}/bin"/>
402 </chmod>
403 <!-- local area -->
404 <copy file="${build.dir}/${final.name}.jar"
405 todir="${runtime.local}/lib"/>
406 <copy todir="${runtime.local}/lib/native">
407 <fileset dir="lib/native"/>
408 </copy>
409 <copy todir="${runtime.local}/conf">
410 <fileset dir="${conf.dir}" excludes="*.template"/>
411 </copy>
412 <copy todir="${runtime.local}/bin">
413 <fileset dir="src/bin"/>
414 </copy>
415 <chmod perm="ugo+x" type="file">
416 <fileset dir="${runtime.local}/bin"/>
417 </chmod>
418 <copy todir="${runtime.local}/lib">
419 <fileset dir="${build.dir}/lib"/>
420 </copy>
421 <copy todir="${runtime.local}/plugins">
422 <fileset dir="${build.dir}/plugins"/>
423 </copy>
424 <copy todir="${runtime.local}/test">
425 <fileset dir="${build.dir}/test"/>
426 </copy>
427 </target>
428
429 <!-- ================================================================== -->
430 <!-- Compile test code -->
431 <!-- ================================================================== -->
432 <target name="compile-core-test" depends="init, compile-core, resolve-test" description="--> compile test code">
433 <javac
434 encoding="${build.encoding}"
435 srcdir="${test.src.dir}"
436 includes="org/apache/nutch/**/*.java org/commoncrawl/**/*.java"
437 destdir="${test.build.classes}"
438 debug="${javac.debug}"
439 optimize="${javac.optimize}"
440 target="${javac.version}"
441 source="${javac.version}"
442 deprecation="${javac.deprecation}">
443 <compilerarg value="-Xlint:-path"/>
444 <classpath refid="test.classpath"/>
445 </javac>
446 </target>
447
448 <!-- ================================================================== -->
449 <!-- Run Nutch proxy -->
450 <!-- ================================================================== -->
451
452 <target name="proxy" depends="compile-core-test, job" description="--> run nutch proxy">
453 <java classname="org.apache.nutch.tools.proxy.ProxyTestbed" fork="true">
454 <classpath refid="test.classpath"/>
455 <arg value="-fake"/>
456 <!--
457 <arg value="-delay"/>
458 <arg value="-200"/>
459 -->
460 <jvmarg line="-Djavax.xml.parsers.DocumentBuilderFactory=com.sun.org.apache.xerces.internal.jaxp.DocumentBuilderFactoryImpl"/>
461 </java>
462 </target>
463
464 <!-- ================================================================== -->
465 <!-- Run Nutch benchmarking analysis -->
466 <!-- ================================================================== -->
467
468 <target name="benchmark" description="--> run nutch benchmarking analysis">
469 <java classname="org.apache.nutch.tools.Benchmark" fork="true">
470 <classpath refid="test.classpath"/>
471 <jvmarg line="-Xmx512m -Djavax.xml.parsers.DocumentBuilderFactory=com.sun.org.apache.xerces.internal.jaxp.DocumentBuilderFactoryImpl"/>
472 <arg value="-maxPerHost"/>
473 <arg value="10"/>
474 <arg value="-seeds"/>
475 <arg value="1"/>
476 <arg value="-depth"/>
477 <arg value="5"/>
478 </java>
479 </target>
480
481 <!-- ================================================================== -->
482 <!-- Run unit tests -->
483 <!-- ================================================================== -->
484 <target name="test" depends="test-core, test-plugins" description="--> run JUnit tests"/>
485
486 <target name="test-core" depends="compile-core-test, job" description="--> run core JUnit tests only">
487
488 <delete dir="${test.build.data}"/>
489 <mkdir dir="${test.build.data}"/>
490 <!--
491 copy resources needed in junit tests
492 -->
493 <copy todir="${test.build.data}">
494 <fileset dir="src/testresources" includes="**/*"/>
495 </copy>
496
497 <copy file="${test.src.dir}/log4j.properties"
498 todir="${test.build.classes}"/>
499
500 <copy file="${test.src.dir}/crawl-tests.xml"
501 todir="${test.build.classes}"/>
502
503 <copy file="${test.src.dir}/domain-urlfilter.txt"
504 todir="${test.build.classes}"/>
505
506 <copy file="${test.src.dir}/filter-all.txt"
507 todir="${test.build.classes}"/>
508
509 <junit printsummary="yes" haltonfailure="no" fork="yes"
510 forkmode="once" dir="${basedir}"
511 errorProperty="tests.failed" failureProperty="tests.failed"
512 maxmemory="1000m">
513 <sysproperty key="test.build.data" value="${test.build.data}"/>
514 <sysproperty key="test.src.dir" value="${test.src.dir}"/>
515 <sysproperty key="javax.xml.parsers.DocumentBuilderFactory" value="com.sun.org.apache.xerces.internal.jaxp.DocumentBuilderFactoryImpl"/>
516 <classpath refid="test.classpath"/>
517 <formatter type="${test.junit.output.format}" />
518 <batchtest todir="${test.build.dir}" unless="testcase">
519 <fileset dir="${test.src.dir}"
520 includes="**/Test*.java" excludes="**/${test.exclude}.java" />
521 </batchtest>
522 <batchtest todir="${test.build.dir}" if="testcase">
523 <fileset dir="${test.src.dir}" includes="**/${testcase}.java"/>
524 </batchtest>
525 </junit>
526
527 <fail if="tests.failed">Tests failed!</fail>
528
529 </target>
530
531 <target name="test-plugins" depends="resolve-test, compile" description="--> run plugin JUnit tests only">
532 <ant dir="src/plugin" target="test" inheritAll="false"/>
533 </target>
534
535 <target name="test-plugin" depends="resolve-test, compile" description="--> run a single plugin's JUnit tests">
536 <ant dir="src/plugin" target="test-single" inheritAll="false"/>
537 </target>
538
539 <target name="nightly" depends="test, tar-src, zip-src" description="--> run the nightly target build">
540 </target>
541
542 <!-- ================================================================== -->
543 <!-- Ivy targets -->
544 <!-- ================================================================== -->
545
546 <!-- target: resolve ================================================= -->
547 <target name="resolve-default" depends="clean-default-lib, init" description="--> resolve and retrieve dependencies with ivy">
548 <ivy:resolve file="${ivy.file}" conf="default" log="download-only"/>
549 <ivy:retrieve pattern="${build.lib.dir}/[artifact]-[revision].[ext]" symlink="false" log="quiet"/>
550 <antcall target="copy-libs"/>
551 </target>
552
553 <target name="resolve-test" depends="clean-test-lib, init" description="--> resolve and retrieve dependencies with ivy">
554 <ivy:resolve file="${ivy.file}" conf="test" log="download-only"/>
555 <ivy:retrieve pattern="${test.build.lib.dir}/[artifact]-[revision].[ext]" symlink="false" log="quiet"/>
556 <antcall target="copy-libs"/>
557 </target>
558
559 <target name="copy-libs" description="--> copy the libs in lib, which are not ivy enabled">
560 <!-- copy the libs in lib, which are not ivy enabled-->
561 <copy todir="${build.lib.dir}/" failonerror="false">
562 <fileset dir="${lib.dir}" includes="**/*.jar"/>
563 </copy>
564 </target>
565
566 <!-- target: publish-local =========================================== -->
567 <target name="publish-local" depends="jar" description="--> publish this project in the local ivy repository">
568 <ivy:publish artifactspattern="${build.dir}/[artifact]-${version}.[ext]"
569 resolver="local"
570 pubrevision="${version}"
571 pubdate="${now}"
572 status="integration"
573 forcedeliver="true"
574 overwrite="true"
575 />
576 <echo message="project ${ant.project.name} published locally with version ${version}" />
577 </target>
578
579 <!-- target: report ================================================== -->
580 <target name="report" depends="resolve-test" description="--> generates a report of dependencies">
581 <ivy:report todir="${build.dir}" xml="true"/>
582 </target>
583
584 <!-- target: 3rd-party licenses report =============================== -->
585 <target name="report-licenses" depends="resolve-default" description="--> generates a report of licenses of dependencies">
586 <ivy:report todir="${build.dir}" xml="false" graph="false" xslfile="ivy/ivy-report-license.xsl"
587 outputpattern="[organisation]-[module]-[conf]-3rd-party-licenses.tsv"/>
588 </target>
589
590 <!-- target: ivy-init ================================================ -->
591 <target name="ivy-init" depends="ivy-probe-antlib, ivy-init-antlib" description="--> initialise Ivy settings">
592 <ivy:settings file="${ivy.dir}/ivysettings.xml" />
593 </target>
594
595 <!-- target: ivy-probe-antlib ======================================== -->
596 <target name="ivy-probe-antlib" description="--> probe the antlib library">
597 <condition property="ivy.found">
598 <typefound uri="antlib:org.apache.ivy.ant" name="cleancache" />
599 </condition>
600 </target>
601
602 <!-- target: ivy-download ============================================ -->
603 <target name="ivy-download" description="--> download ivy">
604 <available file="${ivy.jar}" property="ivy.jar.found"/>
605 <antcall target="ivy-download-unchecked"/>
606 </target>
607
608 <!-- target: ivy-download-unchecked ================================== -->
609 <target name="ivy-download-unchecked" unless="ivy.jar.found" description="--> fetch any ivy file">
610 <get src="${ivy.repo.url}" dest="${ivy.jar}" usetimestamp="true" />
611 </target>
612
613 <!-- target: ivy-init-antlib ========================================= -->
614 <target name="ivy-init-antlib" depends="ivy-download" unless="ivy.found" description="--> attempt to use Ivy with Antlib">
615 <typedef uri="antlib:org.apache.ivy.ant" onerror="fail" loaderRef="ivyLoader">
616 <classpath>
617 <pathelement location="${ivy.jar}" />
618 </classpath>
619 </typedef>
620 <fail>
621 <condition>
622 <not>
623 <typefound uri="antlib:org.apache.ivy.ant" name="cleancache" />
624 </not>
625 </condition>
626 You need Apache Ivy 2.0 or later from http://ant.apache.org/
627 It could not be loaded from ${ivy.repo.url}
628 </fail>
629 </target>
630
631 <!-- Check dependencies for security vulnerabilities -->
632 <!-- requires installation of OWASP dependency check tool, see -->
633 <!-- https://jeremylong.github.io/DependencyCheck/dependency-check-ant/index.html -->
634 <!-- get http://dl.bintray.com/jeremy-long/owasp/dependency-check-ant-3.3.2-release.zip -->
635 <!-- and unzip in directory ./ivy/ -->
636 <property name="dependency-check.home" value="${ivy.dir}/dependency-check-ant/"/>
637 <path id="dependency-check.path">
638 <pathelement location="${dependency-check.home}/dependency-check-ant.jar"/>
639 <fileset dir="${dependency-check.home}/lib" erroronmissingdir="false">
640 <include name="*.jar"/>
641 </fileset>
642 </path>
643 <taskdef resource="dependency-check-taskdefs.properties" onerror="ignore">
644 <classpath refid="dependency-check.path" />
645 </taskdef>
646 <target name="report-vulnerabilities" description="--> check dependencies for security vulnerabilities">
647 <dependency-check projectname="${name}"
648 reportoutputdirectory="${build.dir}"
649 reportformat="ALL">
650 <suppressionfile path="${dependency-check.home}/dependency-check-suppressions.xml" />
651 <retirejsFilter regex="copyright.*jeremy long" />
652 <fileset dir="${build.dir}">
653 <include name="lib/*.jar"/>
654 <include name="plugins/*/*.jar"/>
655 </fileset>
656 </dependency-check>
657 </target>
658
659 <!-- ================================================================== -->
660 <!-- Documentation -->
661 <!-- ================================================================== -->
662 <target name="javadoc" depends="compile" description="--> generate Javadoc">
663 <fail message="Unsupported Java version: ${java.version}. Javadoc requires Java version 7u25 or greater. See https://issues.apache.org/jira/browse/NUTCH-1590">
664 <condition>
665 <or>
666 <matches string="${java.version}" pattern="1.7.0_2[01234].+" casesensitive="false" />
667 <matches string="${java.version}" pattern="1.7.0_1.+" casesensitive="false" />
668 <equals arg1="${ant.java.version}" arg2="1.6" />
669 <equals arg1="${ant.java.version}" arg2="1.5" />
670 </or>
671 </condition>
672 </fail>
673 <mkdir dir="${build.javadoc}"/>
674 <javadoc
675 overview="${src.dir}/overview.html"
676 destdir="${build.javadoc}"
677 author="true"
678 version="true"
679 use="true"
680 windowtitle="${name} ${version} API"
681 doctitle="${name} ${version} API"
682 bottom="Copyright &amp;copy; ${year} The Apache Software Foundation"
683 >
684 <arg value="${javadoc.proxy.host}"/>
685 <arg value="${javadoc.proxy.port}"/>
686
687 <packageset dir="${src.dir}"/>
688 <packageset dir="${plugins.dir}/any23/src/java/" />
689 <packageset dir="${plugins.dir}/creativecommons/src/java"/>
690 <packageset dir="${plugins.dir}/feed/src/java"/>
691 <packageset dir="${plugins.dir}/headings/src/java"/>
692 <packageset dir="${plugins.dir}/exchange-jexl/src/java"/>
693 <packageset dir="${plugins.dir}/index-anchor/src/java"/>
694 <packageset dir="${plugins.dir}/index-basic/src/java"/>
695 <packageset dir="${plugins.dir}/index-geoip/src/java"/>
696 <packageset dir="${plugins.dir}/index-jexl-filter/src/java"/>
697 <packageset dir="${plugins.dir}/index-links/src/java"/>
698 <packageset dir="${plugins.dir}/index-metadata/src/java"/>
699 <packageset dir="${plugins.dir}/index-more/src/java"/>
700 <packageset dir="${plugins.dir}/index-replace/src/java"/>
701 <packageset dir="${plugins.dir}/index-static/src/java"/>
702 <packageset dir="${plugins.dir}/indexer-cloudsearch/src/java/" />
703 <packageset dir="${plugins.dir}/indexer-csv/src/java"/>
704 <packageset dir="${plugins.dir}/indexer-dummy/src/java"/>
705 <packageset dir="${plugins.dir}/indexer-elastic-rest/src/java/"/>
706 <packageset dir="${plugins.dir}/indexer-elastic/src/java/" />
707 <packageset dir="${plugins.dir}/indexer-kafka/src/java/" />
708 <packageset dir="${plugins.dir}/indexer-rabbit/src/java"/>
709 <packageset dir="${plugins.dir}/indexer-solr/src/java"/>
710 <packageset dir="${plugins.dir}/language-identifier/src/java"/>
711 <packageset dir="${plugins.dir}/lib-htmlunit/src/java"/>
712 <packageset dir="${plugins.dir}/lib-http/src/java"/>
713 <packageset dir="${plugins.dir}/lib-rabbitmq/src/java"/>
714 <packageset dir="${plugins.dir}/lib-regex-filter/src/java"/>
715 <packageset dir="${plugins.dir}/lib-selenium/src/java"/>
716 <packageset dir="${plugins.dir}/microformats-reltag/src/java"/>
717 <packageset dir="${plugins.dir}/mimetype-filter/src/java"/>
718 <packageset dir="${plugins.dir}/parse-ext/src/java"/>
719 <packageset dir="${plugins.dir}/parse-html/src/java"/>
720 <packageset dir="${plugins.dir}/parse-js/src/java"/>
721 <packageset dir="${plugins.dir}/parse-metatags/src/java"/>
722 <packageset dir="${plugins.dir}/parse-swf/src/java"/>
723 <packageset dir="${plugins.dir}/parse-tika/src/java"/>
724 <packageset dir="${plugins.dir}/parse-zip/src/java"/>
725 <packageset dir="${plugins.dir}/parsefilter-naivebayes/src/java"/>
726 <packageset dir="${plugins.dir}/parsefilter-regex/src/java"/>
727 <packageset dir="${plugins.dir}/protocol-file/src/java"/>
728 <packageset dir="${plugins.dir}/protocol-ftp/src/java"/>
729 <packageset dir="${plugins.dir}/protocol-htmlunit/src/java"/>
730 <packageset dir="${plugins.dir}/protocol-http/src/java"/>
731 <packageset dir="${plugins.dir}/protocol-httpclient/src/java"/>
732 <packageset dir="${plugins.dir}/protocol-interactiveselenium/src/java"/>
733 <packageset dir="${plugins.dir}/protocol-okhttp/src/java"/>
734 <packageset dir="${plugins.dir}/protocol-selenium/src/java"/>
735 <packageset dir="${plugins.dir}/publish-rabbitmq/src/java"/>
736 <packageset dir="${plugins.dir}/scoring-adaptive/src/java"/>
737 <packageset dir="${plugins.dir}/scoring-depth/src/java"/>
738 <packageset dir="${plugins.dir}/scoring-link/src/java"/>
739 <packageset dir="${plugins.dir}/scoring-opic/src/java"/>
740 <packageset dir="${plugins.dir}/scoring-orphan/src/java"/>
741 <packageset dir="${plugins.dir}/scoring-similarity/src/java"/>
742 <packageset dir="${plugins.dir}/subcollection/src/java"/>
743 <packageset dir="${plugins.dir}/tld/src/java"/>
744 <packageset dir="${plugins.dir}/urlfilter-automaton/src/java"/>
745 <packageset dir="${plugins.dir}/urlfilter-domain/src/java"/>
746 <packageset dir="${plugins.dir}/urlfilter-domainblacklist/src/java"/>
747 <packageset dir="${plugins.dir}/urlfilter-fast/src/java"/>
748 <packageset dir="${plugins.dir}/urlfilter-ignoreexempt/src/java"/>
749 <packageset dir="${plugins.dir}/urlfilter-prefix/src/java"/>
750 <packageset dir="${plugins.dir}/urlfilter-regex/src/java"/>
751 <packageset dir="${plugins.dir}/urlfilter-suffix/src/java"/>
752 <packageset dir="${plugins.dir}/urlfilter-validator/src/java"/>
753 <packageset dir="${plugins.dir}/urlmeta/src/java"/>
754 <packageset dir="${plugins.dir}/urlnormalizer-ajax/src/java"/>
755 <packageset dir="${plugins.dir}/urlnormalizer-basic/src/java"/>
756 <packageset dir="${plugins.dir}/urlnormalizer-host/src/java"/>
757 <packageset dir="${plugins.dir}/urlnormalizer-pass/src/java"/>
758 <packageset dir="${plugins.dir}/urlnormalizer-protocol/src/java"/>
759 <packageset dir="${plugins.dir}/urlnormalizer-querystring/src/java"/>
760 <packageset dir="${plugins.dir}/urlnormalizer-regex/src/java"/>
761 <packageset dir="${plugins.dir}/urlnormalizer-slash/src/java"/>
762
763 <link href="${javadoc.link.java}"/>
764 <link href="${javadoc.link.hadoop}"/>
765 <link href="${javadoc.link.lucene.core}"/>
766 <link href="${javadoc.link.lucene.analyzers-common}"/>
767 <link href="${javadoc.link.solr-solrj}"/>
768
769 <classpath refid="classpath"/>
770 <classpath>
771 <fileset dir="${plugins.dir}" >
772 <include name="**/*.jar"/>
773 </fileset>
774 </classpath>
775
776 <group title="Core" packages="org.apache.nutch.*"/>
777 <group title="Plugins API" packages="${plugins.api}"/>
778 <group title="Protocol Plugins" packages="${plugins.protocol}"/>
779 <group title="URL Filter Plugins" packages="${plugins.urlfilter}"/>
780 <group title="URL Normalizer Plugins" packages="${plugins.urlnormalizer}"/>
781 <group title="Scoring Plugins" packages="${plugins.scoring}"/>
782 <group title="Parse Plugins" packages="${plugins.parse}"/>
783 <group title="Parse Filter Plugins" packages="${plugins.parsefilter}"/>
784 <group title="Publisher Plugins" packages="${plugins.publisher}"/>
785 <group title="Exchange Plugins" packages="${plugins.exchange}"/>
786 <group title="Indexing Filter Plugins" packages="${plugins.index}"/>
787 <group title="Indexer Plugins" packages="${plugins.indexer}"/>
788 <group title="Misc. Plugins" packages="${plugins.misc}"/>
789 </javadoc>
790 <!-- Copy the plugin.dtd file to the plugin doc-files dir -->
791 <copy file="${plugins.dir}/plugin.dtd"
792 todir="${build.javadoc}/org/apache/nutch/plugin/doc-files"/>
793 </target>
794
795 <target name="default-doc" description="--> generate default Nutch documentation">
796 <style basedir="${conf.dir}" destdir="${docs.dir}"
797 includes="nutch-default.xml" style="conf/nutch-conf.xsl"/>
798 </target>
799
800 <!-- ================================================================== -->
801 <!-- D I S T R I B U T I O N -->
802 <!-- ================================================================== -->
803 <!-- -->
804 <!-- ================================================================== -->
805 <target name="package-src" depends="runtime, javadoc" description="--> generate source distribution package">
806 <mkdir dir="${dist.dir}"/>
807 <mkdir dir="${src.dist.version.dir}"/>
808 <mkdir dir="${src.dist.version.dir}/lib"/>
809 <mkdir dir="${src.dist.version.dir}/docs"/>
810 <mkdir dir="${src.dist.version.dir}/docs/api"/>
811 <mkdir dir="${src.dist.version.dir}/ivy"/>
812
813 <copy todir="${src.dist.version.dir}/lib" includeEmptyDirs="false">
814 <fileset dir="lib"/>
815 </copy>
816
817 <copy todir="${src.dist.version.dir}/conf">
818 <fileset dir="${conf.dir}" excludes="**/*.template"/>
819 </copy>
820
821 <copy todir="${src.dist.version.dir}/docs/api">
822 <fileset dir="${build.javadoc}"/>
823 </copy>
824
825 <copy todir="${src.dist.version.dir}">
826 <fileset dir=".">
827 <include name="*.txt" />
828 <!--<include name="KEYS" />-->
829 </fileset>
830 </copy>
831
832 <copy todir="${src.dist.version.dir}/src" includeEmptyDirs="true">
833 <fileset dir="src"/>
834 </copy>
835
836 <copy todir="${src.dist.version.dir}/ivy" includeEmptyDirs="true">
837 <fileset dir="ivy"/>
838 </copy>
839
840 <copy todir="${src.dist.version.dir}/" file="build.xml"/>
841 <copy todir="${src.dist.version.dir}/" file="default.properties"/>
842
843 </target>
844
845 <target name="package-bin" depends="runtime, javadoc" description="--> generate binary distribution package">
846 <mkdir dir="${dist.dir}"/>
847 <mkdir dir="${bin.dist.version.dir}"/>
848 <mkdir dir="${bin.dist.version.dir}/lib"/>
849 <mkdir dir="${bin.dist.version.dir}/bin"/>
850 <mkdir dir="${bin.dist.version.dir}/conf"/>
851 <mkdir dir="${bin.dist.version.dir}/docs"/>
852 <mkdir dir="${bin.dist.version.dir}/docs/api"/>
853 <mkdir dir="${bin.dist.version.dir}/plugins"/>
854
855 <copy todir="${bin.dist.version.dir}/lib" includeEmptyDirs="false">
856 <fileset dir="runtime/local/lib"/>
857 </copy>
858
859 <copy todir="${bin.dist.version.dir}/bin">
860 <fileset dir="runtime/local/bin"/>
861 </copy>
862
863 <chmod perm="ugo+x" type="file">
864 <fileset dir="${bin.dist.version.dir}/bin"/>
865 </chmod>
866
867 <copy todir="${bin.dist.version.dir}/conf">
868 <fileset dir="runtime/local/conf" excludes="**/*.template"/>
869 </copy>
870
871 <copy todir="${bin.dist.version.dir}/docs/api">
872 <fileset dir="${build.javadoc}"/>
873 </copy>
874
875 <copy todir="${bin.dist.version.dir}">
876 <fileset dir=".">
877 <include name="*.txt" />
878 </fileset>
879 </copy>
880
881 <copy todir="${bin.dist.version.dir}/plugins" includeEmptyDirs="true">
882 <fileset dir="runtime/local/plugins"/>
883 </copy>
884
885 </target>
886
887 <!-- ================================================================== -->
888 <!-- Make src release tarball -->
889 <!-- ================================================================== -->
890 <target name="tar-src" depends="package-src" description="--> generate src.tar.gz distribution package">
891 <tar compression="gzip" longfile="gnu"
892 destfile="${src.dist.version.dir}.tar.gz">
893 <tarfileset dir="${src.dist.version.dir}" mode="664" prefix="${final.name}">
894 <exclude name="src/bin/*" />
895 <include name="**" />
896 </tarfileset>
897 <tarfileset dir="${src.dist.version.dir}" mode="755" prefix="${final.name}">
898 <include name="src/bin/*" />
899 </tarfileset>
900 </tar>
901 </target>
902
903 <!-- ================================================================== -->
904 <!-- Make bin release tarball -->
905 <!-- ================================================================== -->
906 <target name="tar-bin" depends="package-bin" description="--> generate bin.tar.gz distribution package">
907 <tar compression="gzip" longfile="gnu"
908 destfile="${bin.dist.version.dir}.tar.gz">
909 <tarfileset dir="${bin.dist.version.dir}" mode="664" prefix="${final.name}">
910 <exclude name="bin/*" />
911 <include name="**" />
912 </tarfileset>
913 <tarfileset dir="${bin.dist.version.dir}" mode="755" prefix="${final.name}">
914 <include name="bin/*" />
915 </tarfileset>
916 </tar>
917 </target>
918
919 <!-- ================================================================== -->
920 <!-- Make src release zip -->
921 <!-- ================================================================== -->
922 <target name="zip-src" depends="package-src" description="--> generate src.zip distribution package">
923 <zip compress="true" casesensitive="yes"
924 destfile="${src.dist.version.dir}.zip">
925 <zipfileset dir="${src.dist.version.dir}" filemode="664" prefix="${final.name}">
926 <exclude name="src/bin/*" />
927 <include name="**" />
928 </zipfileset>
929 <zipfileset dir="${src.dist.version.dir}" filemode="755" prefix="${final.name}">
930 <include name="src/bin/*" />
931 </zipfileset>
932 </zip>
933 </target>
934
935 <!-- ================================================================== -->
936 <!-- Make bin release zip -->
937 <!-- ================================================================== -->
938 <target name="zip-bin" depends="package-bin" description="--> generate bin.zip distribution package">
939 <zip compress="true" casesensitive="yes"
940 destfile="${bin.dist.version.dir}.zip">
941 <zipfileset dir="${bin.dist.version.dir}" filemode="664" prefix="${final.name}">
942 <exclude name="bin/*" />
943 <include name="**" />
944 </zipfileset>
945 <zipfileset dir="${bin.dist.version.dir}" filemode="755" prefix="${final.name}">
946 <include name="bin/*" />
947 </zipfileset>
948 </zip>
949 </target>
950
951 <!-- ================================================================== -->
952 <!-- Clean. Delete the build files, and their directories -->
953 <!-- ================================================================== -->
954
955 <!-- target: clean =================================================== -->
956 <target name="clean" depends="clean-build, clean-lib, clean-dist, clean-runtime" description="--> clean the project" />
957
958 <!-- target: clean-eclipse ============================================ -->
959 <target name="clean-eclipse" depends="clean-build, clean-lib, clean-dist" description="--> cleans the eclipse project"/>
960
961 <!-- target: clean-local ============================================= -->
962 <target name="clean-local" depends=""
963 description="--> cleans the local repository for the current module">
964 <delete dir="${ivy.local.default.root}/${ivy.organisation}/${ivy.module}"/>
965 </target>
966
967 <!-- target: clean-lib =============================================== -->
968 <target name="clean-lib" depends="clean-default-lib, clean-test-lib"
969 description="--> clean the project libraries directories (dependencies: default + test)">
970 </target>
971 <!-- target: clean-default-lib =============================================== -->
972 <target name="clean-default-lib" description="--> clean the project libraries directory (dependencies)">
973 <delete includeemptydirs="true" dir="${build.lib.dir}"/>
974 </target>
975 <!-- target: clean-test-lib =============================================== -->
976 <target name="clean-test-lib" description="--> clean the project test libraries directory (dependencies)">
977 <delete includeemptydirs="true" dir="${test.build.lib.dir}"/>
978 </target>
979
980 <!-- target: clean-build ============================================= -->
981 <target name="clean-build" description="--> clean the project built files">
982 <delete includeemptydirs="true" dir="${build.dir}"/>
983 </target>
984
985 <!-- target: clean-dist ============================================= -->
986 <target name="clean-dist" description="--> clean the project dist files">
987 <delete includeemptydirs="true" dir="${dist.dir}"/>
988 </target>
989
990 <!-- target: clean-cache ============================================= -->
991 <target name="clean-cache" depends=""
992 description="--> delete ivy cache">
993 <ivy:cleancache />
994 </target>
995
996 <target name="clean-runtime" description="--> clean the project runtime area">
997 <delete includeemptydirs="true" dir="${runtime.dir}"/>
998 </target>
999
1000 <!-- ================================================================== -->
1001 <!-- RAT targets -->
1002 <!-- ================================================================== -->
1003 <target name="rat-sources-typedef" description="--> run RAT antlib task">
1004 <typedef resource="org/apache/rat/anttasks/antlib.xml" >
1005 <classpath>
1006 <fileset dir="." includes="rat*.jar"/>
1007 </classpath>
1008 </typedef>
1009 </target>
1010
1011 <target name="rat-sources" depends="rat-sources-typedef"
1012 description="--> runs RAT tasks over src/java">
1013 <rat:report xmlns:rat="antlib:org.apache.rat.anttasks">
1014 <fileset dir="src">
1015 <include name="java/**/*"/>
1016 <include name="plugin/**/src/**/*"/>
1017 </fileset>
1018 </rat:report>
1019 </target>
1020
1021
1022 <!-- ================================================================== -->
1023 <!-- Eclipse targets -->
1024 <!-- ================================================================== -->
1025
1026 <!-- classpath for generating eclipse project -->
1027 <path id="eclipse.classpath">
1028 <fileset dir="${build.lib.dir}">
1029 <include name="*.jar" />
1030 <exclude name="ant-eclipse-1.0-jvm1.2.jar" />
1031 </fileset>
1032 <fileset dir="${build.plugins}">
1033 <include name="**/*.jar" />
1034 </fileset>
1035 <fileset dir="${test.build.lib.dir}">
1036 <include name="*.jar" />
1037 </fileset>
1038 </path>
1039
1040 <!-- target: ant-eclipse-download =================================== -->
1041 <target name="ant-eclipse-download" description="--> downloads the ant-eclipse binary.">
1042 <get src="https://downloads.sourceforge.net/project/ant-eclipse/ant-eclipse/1.0/ant-eclipse-1.0.bin.tar.bz2"
1043 dest="${build.dir}/ant-eclipse-1.0.bin.tar.bz2" usetimestamp="false" />
1044
1045 <untar src="${build.dir}/ant-eclipse-1.0.bin.tar.bz2"
1046 dest="${build.dir}" compression="bzip2">
1047 <patternset>
1048 <include name="lib/ant-eclipse-1.0-jvm1.2.jar"/>
1049 </patternset>
1050 </untar>
1051
1052 <delete file="${build.dir}/ant-eclipse-1.0.bin.tar.bz2" />
1053 </target>
1054
1055 <!-- target: eclipse ================================================ -->
1056 <target name="eclipse"
1057 depends="clean-eclipse,init,resolve-test,job,ant-eclipse-download"
1058 description="--> create eclipse project files">
1059
1060 <pathconvert property="eclipse.project">
1061 <path path="${basedir}"/>
1062 <regexpmapper from="^.*/([^/]+)$$" to="\1" handledirsep="yes"/>
1063 </pathconvert>
1064
1065 <taskdef name="eclipse"
1066 classname="prantl.ant.eclipse.EclipseTask"
1067 classpath="${build.dir}/lib/ant-eclipse-1.0-jvm1.2.jar" />
1068 <eclipse updatealways="true">
1069 <project name="${eclipse.project}" />
1070 <classpath>
1071 <library path="${conf.dir}" exported="false" />
1072 <library path="${basedir}/src/bin" exported="false" />
1073 <library pathref="eclipse.classpath" exported="false" />
1074
1075 <source path="${basedir}/src/java/" />
1076 <source path="${basedir}/src/test/" output="build/test/classes" />
1077
1078 <source path="${plugins.dir}/any23/src/java/" />
1079 <source path="${plugins.dir}/any23/src/test/" />
1080 <source path="${plugins.dir}/creativecommons/src/java/" />
1081 <source path="${plugins.dir}/creativecommons/src/test/" />
1082 <source path="${plugins.dir}/feed/src/java/" />
1083 <source path="${plugins.dir}/feed/src/test/" />
1084 <source path="${plugins.dir}/headings/src/java/" />
1085 <source path="${plugins.dir}/headings/src/test/" />
1086 <source path="${plugins.dir}/exchange-jexl/src/java/" />
1087 <source path="${plugins.dir}/index-anchor/src/java/" />
1088 <source path="${plugins.dir}/index-anchor/src/test/" />
1089 <source path="${plugins.dir}/index-basic/src/java/" />
1090 <source path="${plugins.dir}/index-basic/src/test/" />
1091 <source path="${plugins.dir}/index-geoip/src/java/" />
1092 <source path="${plugins.dir}/index-jexl-filter/src/java/" />
1093 <source path="${plugins.dir}/index-jexl-filter/src/test/" />
1094 <source path="${plugins.dir}/index-links/src/java/" />
1095 <source path="${plugins.dir}/index-links/src/test/" />
1096 <source path="${plugins.dir}/index-metadata/src/java/" />
1097 <source path="${plugins.dir}/index-more/src/java/" />
1098 <source path="${plugins.dir}/index-more/src/test/" />
1099 <source path="${plugins.dir}/index-replace/src/java/" />
1100 <source path="${plugins.dir}/index-replace/src/test/" />
1101 <source path="${plugins.dir}/index-static/src/java/" />
1102 <source path="${plugins.dir}/index-static/src/test/" />
1103 <source path="${plugins.dir}/indexer-cloudsearch/src/java/" />
1104 <source path="${plugins.dir}/indexer-csv/src/java"/>
1105 <source path="${plugins.dir}/indexer-csv/src/test"/>
1106 <source path="${plugins.dir}/indexer-dummy/src/java/" />
1107 <source path="${plugins.dir}/indexer-elastic-rest/src/java/"/>
1108 <source path="${plugins.dir}/indexer-elastic/src/java/" />
1109 <source path="${plugins.dir}/indexer-kafka/src/java/" />
1110 <source path="${plugins.dir}/indexer-rabbit/src/java/" />
1111 <source path="${plugins.dir}/indexer-solr/src/java/" />
1112 <source path="${plugins.dir}/language-identifier/src/java/" />
1113 <source path="${plugins.dir}/language-identifier/src/test/" />
1114 <source path="${plugins.dir}/lib-htmlunit/src/java/" />
1115 <source path="${plugins.dir}/lib-http/src/java/" />
1116 <source path="${plugins.dir}/lib-http/src/test/" />
1117 <source path="${plugins.dir}/lib-rabbitmq/src/java/" />
1118 <source path="${plugins.dir}/lib-regex-filter/src/java/" />
1119 <source path="${plugins.dir}/lib-regex-filter/src/test/" />
1120 <source path="${plugins.dir}/lib-selenium/src/java/" />
1121 <source path="${plugins.dir}/microformats-reltag/src/java/" />
1122 <source path="${plugins.dir}/mimetype-filter/src/java/" />
1123 <source path="${plugins.dir}/mimetype-filter/src/test/" />
1124 <source path="${plugins.dir}/parse-ext/src/java/" />
1125 <source path="${plugins.dir}/parse-ext/src/test/" />
1126 <source path="${plugins.dir}/parse-html/src/java/" />
1127 <source path="${plugins.dir}/parse-html/src/test/" />
1128 <source path="${plugins.dir}/parse-js/src/java/" />
1129 <source path="${plugins.dir}/parse-js/src/test/" />
1130 <source path="${plugins.dir}/parse-metatags/src/java/" />
1131 <source path="${plugins.dir}/parse-metatags/src/test/" />
1132 <source path="${plugins.dir}/parse-swf/src/java/" />
1133 <source path="${plugins.dir}/parse-swf/src/test/" />
1134 <source path="${plugins.dir}/parse-tika/src/java/" />
1135 <source path="${plugins.dir}/parse-tika/src/test/" />
1136 <source path="${plugins.dir}/parse-zip/src/java/" />
1137 <source path="${plugins.dir}/parse-zip/src/test/" />
1138 <source path="${plugins.dir}/parsefilter-naivebayes/src/java/" />
1139 <source path="${plugins.dir}/parsefilter-regex/src/java/" />
1140 <source path="${plugins.dir}/parsefilter-regex/src/test/" />
1141 <source path="${plugins.dir}/protocol-file/src/java/" />
1142 <source path="${plugins.dir}/protocol-file/src/test/" />
1143 <source path="${plugins.dir}/protocol-ftp/src/java/" />
1144 <source path="${plugins.dir}/protocol-htmlunit/src/java/" />
1145 <source path="${plugins.dir}/protocol-http/src/java/" />
1146 <source path="${plugins.dir}/protocol-http/src/test/" />
1147 <source path="${plugins.dir}/protocol-httpclient/src/java/" />
1148 <source path="${plugins.dir}/protocol-httpclient/src/test/" />
1149 <source path="${plugins.dir}/protocol-interactiveselenium/src/java/" />
1150 <source path="${plugins.dir}/protocol-okhttp/src/java/" />
1151 <source path="${plugins.dir}/protocol-okhttp/src/test/" />
1152 <source path="${plugins.dir}/protocol-selenium/src/java"/>
1153 <source path="${plugins.dir}/publish-rabbitmq/src/java"/>
1154 <source path="${plugins.dir}/scoring-adaptive/src/java"/>
1155 <source path="${plugins.dir}/scoring-depth/src/java/" />
1156 <source path="${plugins.dir}/scoring-link/src/java/" />
1157 <source path="${plugins.dir}/scoring-opic/src/java/" />
1158 <source path="${plugins.dir}/scoring-orphan/src/java"/>
1159 <source path="${plugins.dir}/scoring-orphan/src/test"/>
1160 <source path="${plugins.dir}/scoring-similarity/src/java/" />
1161 <source path="${plugins.dir}/subcollection/src/java/" />
1162 <source path="${plugins.dir}/subcollection/src/test/" />
1163 <source path="${plugins.dir}/tld/src/java/" />
1164 <source path="${plugins.dir}/urlfilter-automaton/src/java/" />
1165 <source path="${plugins.dir}/urlfilter-automaton/src/test/" />
1166 <source path="${plugins.dir}/urlfilter-domain/src/java/" />
1167 <source path="${plugins.dir}/urlfilter-domain/src/test/" />
1168 <source path="${plugins.dir}/urlfilter-domainblacklist/src/java/" />
1169 <source path="${plugins.dir}/urlfilter-domainblacklist/src/test/" />
1170 <source path="${plugins.dir}/urlfilter-fast/src/java/"/>
1171 <source path="${plugins.dir}/urlfilter-fast/src/test/"/>
1172 <source path="${plugins.dir}/urlfilter-ignoreexempt/src/java/" />
1173 <source path="${plugins.dir}/urlfilter-prefix/src/java/" />
1174 <source path="${plugins.dir}/urlfilter-prefix/src/test/" />
1175 <source path="${plugins.dir}/urlfilter-regex/src/java/" />
1176 <source path="${plugins.dir}/urlfilter-regex/src/test/" />
1177 <source path="${plugins.dir}/urlfilter-suffix/src/java/" />
1178 <source path="${plugins.dir}/urlfilter-suffix/src/test/" />
1179 <source path="${plugins.dir}/urlfilter-validator/src/java/" />
1180 <source path="${plugins.dir}/urlfilter-validator/src/test/" />
1181 <source path="${plugins.dir}/urlmeta/src/java/" />
1182 <source path="${plugins.dir}/urlnormalizer-ajax/src/java/" />
1183 <source path="${plugins.dir}/urlnormalizer-ajax/src/test/" />
1184 <source path="${plugins.dir}/urlnormalizer-basic/src/java/" />
1185 <source path="${plugins.dir}/urlnormalizer-basic/src/test/" />
1186 <source path="${plugins.dir}/urlnormalizer-host/src/java/" />
1187 <source path="${plugins.dir}/urlnormalizer-host/src/test/" />
1188 <source path="${plugins.dir}/urlnormalizer-pass/src/java/" />
1189 <source path="${plugins.dir}/urlnormalizer-pass/src/test/" />
1190 <source path="${plugins.dir}/urlnormalizer-protocol/src/java/" />
1191 <source path="${plugins.dir}/urlnormalizer-protocol/src/test/" />
1192 <source path="${plugins.dir}/urlnormalizer-querystring/src/java/" />
1193 <source path="${plugins.dir}/urlnormalizer-querystring/src/test/" />
1194 <source path="${plugins.dir}/urlnormalizer-regex/src/java/" />
1195 <source path="${plugins.dir}/urlnormalizer-regex/src/test/" />
1196 <source path="${plugins.dir}/urlnormalizer-slash/src/java/" />
1197 <source path="${plugins.dir}/urlnormalizer-slash/src/test/" />
1198
1199 <output path="${build.classes}" />
1200 </classpath>
1201 </eclipse>
1202 </target>
1203
1204 </project>