001/* 002 * #%L 003 * Netarchivesuite - harvester 004 * %% 005 * Copyright (C) 2005 - 2014 The Royal Danish Library, the Danish State and University Library, 006 * the National Library of France and the Austrian National Library. 007 * %% 008 * This program is free software: you can redistribute it and/or modify 009 * it under the terms of the GNU Lesser General Public License as 010 * published by the Free Software Foundation, either version 2.1 of the 011 * License, or (at your option) any later version. 012 * 013 * This program is distributed in the hope that it will be useful, 014 * but WITHOUT ANY WARRANTY; without even the implied warranty of 015 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 016 * GNU General Lesser Public License for more details. 017 * 018 * You should have received a copy of the GNU General Lesser Public 019 * License along with this program. If not, see 020 * <http://www.gnu.org/licenses/lgpl-2.1.html>. 021 * #L% 022 */ 023package dk.netarkivet.harvester; 024 025import java.util.regex.Pattern; 026 027import dk.netarkivet.common.utils.Settings; 028import dk.netarkivet.harvester.harvesting.distribute.HarvesterReadyMessage; 029import dk.netarkivet.harvester.harvesting.report.HarvestReport; 030 031/** Settings specific to the harvester module of NetarchiveSuite. */ 032public class HarvesterSettings { 033 034 /** The default place in classpath where the settings file can be found. */ 035 private static final String DEFAULT_SETTINGS_CLASSPATH = "dk/netarkivet/harvester/settings.xml"; 036 037 /* 038 * The static initialiser is called when the class is loaded. It will add default values for all settings defined in 039 * this class, by loading them from a settings.xml file in classpath. 040 */ 041 static { 042 Settings.addDefaultClasspathSettings(DEFAULT_SETTINGS_CLASSPATH); 043 } 044 045 // NOTE: The constants defining setting names below are left non-final on 046 // purpose! Otherwise, the static initialiser that loads default values 047 // will not run. 048 049 /** 050 * <b>settings.harvester.datamodel.domain.defaultSeedlist</b>: <br> 051 * Default name of the seedlist to use when new domains are created. 052 */ 053 public static String DEFAULT_SEEDLIST = "settings.harvester.datamodel.domain.defaultSeedlist"; 054 055 /** 056 * <b>settings.harvester.datamodel.domain.validSeedRegex</b>: <br> 057 * Regular expression used to validate a seed within a seedlist. 058 * <p> 059 * Default value accepts all non-empty strings. 060 */ 061 public static String VALID_SEED_REGEX = "settings.harvester.datamodel.domain.validSeedRegex"; 062 063 /** 064 * <b>settings.harvester.datamodel.domain.defaultConfig</b>: <br> 065 * The name of a configuration that is created by default and which is initially used for snapshot harvests. 066 */ 067 public static String DOMAIN_DEFAULT_CONFIG = "settings.harvester.datamodel.domain.defaultConfig"; 068 069 /** 070 * <b>settings.harvester.datamodel.domain.defaultOrderxml</b>: <br> 071 * Name of order xml template used for domains if nothing else is specified. The newly created configurations use 072 * this. This template must exist before harvesting can commence 073 */ 074 public static String DOMAIN_DEFAULT_ORDERXML = "settings.harvester.datamodel.domain.defaultOrderxml"; 075 076 /** 077 * <b>settings.harvester.datamodel.domain.defaultMaxrate</b>: <br> 078 * Default download rate for domain configuration. Not currently enforced. 079 */ 080 public static String DOMAIN_CONFIG_MAXRATE = "settings.harvester.datamodel.domain.defaultMaxrate"; 081 082 /** 083 * <b>settings.harvester.datamodel.domain.defaultMaxbytes</b>: <br> 084 * Default byte limit for domain configuration. 085 */ 086 public static String DOMAIN_CONFIG_MAXBYTES = "settings.harvester.datamodel.domain.defaultMaxbytes"; 087 088 /** 089 * <b>settings.harvester.datamodel.domain.defaultMaxobjects</b>: <br> 090 * Default object limit for domain configuration. 091 */ 092 public static String DOMAIN_CONFIG_MAXOBJECTS = "settings.harvester.datamodel.domain.defaultMaxobjects"; 093 094 /** 095 * <b>settings.harvester.scheduler.jobGen.config.errorFactorPrevResult</b>: <br> 096 * Used when calculating expected size of a harvest of some configuration during job-creation process. This defines 097 * how great a possible factor we will permit a harvest to be larger then the expectation, when basing the 098 * expectation on a previous completed job. 099 */ 100 public static String ERRORFACTOR_PERMITTED_PREVRESULT = "settings.harvester.scheduler.jobGen.config.errorFactorPrevResult"; 101 102 /** 103 * <b>settings.harvester.scheduler.jobGen.config.errorFactorBestGuess</b>: <br> 104 * Used when calculating expected size of a harvest of some configuration during job-creation process. This defines 105 * how great a possible factor we will permit a harvest to be larger then the expectation, when basing the 106 * expectation on previous uncompleted harvests or no harvest data at all. 107 */ 108 public static String ERRORFACTOR_PERMITTED_BESTGUESS = "settings.harvester.scheduler.jobGen.config.errorFactorBestGuess"; 109 110 /** 111 * <b>settings.harvester.scheduler.jobGen.config.expectedAverageBytesPerObject</b>: <br> 112 * How many bytes the average object is expected to be on domains where we don't know any better. This number should 113 * grow over time, as of end of 2005 empirical data shows 38000. 114 */ 115 public static String EXPECTED_AVERAGE_BYTES_PER_OBJECT = "settings.harvester.scheduler.jobGen.config.expectedAverageBytesPerObject"; 116 117 /** 118 * <b>settings.harvester.scheduler.jobGen.config.maxDomainSize</b>: <br> 119 * The initial guess of the domain size (number of objects) of an unknown domain. 120 */ 121 public static String MAX_DOMAIN_SIZE = "settings.harvester.scheduler.jobGen.config.maxDomainSize"; 122 123 /** 124 * <b>settings.harvester.scheduler.jobGen.config.maxRelativeSizeDifference</b>: <br> 125 * The maximum allowed relative difference in expected number of objects retrieved in a single job definition. To 126 * avoid job splitting, set the value as Long.MAX_VALUE. 127 */ 128 public static String JOBS_MAX_RELATIVE_SIZE_DIFFERENCE = "settings.harvester.scheduler.jobGen.config.maxRelativeSizeDifference"; 129 130 /** 131 * <b>settings.harvester.scheduler.jobGen.config.minAbsoluteSizeDifference</b>: <br> 132 * Size differences for jobs below this threshold are ignored, regardless of the limits for the relative size 133 * difference. To avoid job splitting, set the value as Long.MAX_VALUE. 134 */ 135 public static String JOBS_MIN_ABSOLUTE_SIZE_DIFFERENCE = "settings.harvester.scheduler.jobGen.config.minAbsoluteSizeDifference"; 136 137 /** 138 * <b>settings.harvester.scheduler.jobGen.config.maxTotalSize</b>: <br> 139 * When this limit is exceeded no more configurations may be added to a job. To avoid job splitting, set the value 140 * as Long.MAX_VALUE. 141 */ 142 public static String JOBS_MAX_TOTAL_JOBSIZE = "settings.harvester.scheduler.jobGen.config.maxTotalSize"; 143 144 /** 145 * <b>settings.harvester.scheduler.jobGen.maxTimeToCompleteJob</b>: <br> 146 * The limit on how many seconds Heritrix should continue on each job. O means no limit. 147 */ 148 public static String JOBS_MAX_TIME_TO_COMPLETE = "settings.harvester.scheduler.jobGen.maxTimeToCompleteJob"; 149 150 /** 151 * <b>settings.harvester.scheduler.jobGen.domainConfigSubsetSize</b>: <br> 152 * How many domain configurations we will process in one go before making jobs out of them. This amount of domains 153 * will be stored in memory at the same time. To avoid job splitting, set this value as Long.MAX_VALUE. 154 */ 155 public static String JOBGEN_DOMAIN_CONFIG_SUBSET_SIZE = "settings.harvester.scheduler.jobGen.domainConfigSubsetSize"; 156 157 /** 158 * <b>settings.harvester.scheduler.jobGen.config.fixedDomainCountFocused</b>: <br> 159 * If the job generator is {@link FixedDomainConfigurationCountJobGenerator}, then this parameter represents the 160 * maximum number of domain configurations in a partial harvest job. 161 */ 162 public static String JOBGEN_FIXED_CONFIG_COUNT_FOCUSED = "settings.harvester.scheduler.jobGen.config.fixedDomainCountFocused"; 163 164 /** 165 * <b>settings.harvester.scheduler.jobGen.config.fixedDomainCountSnapshot</b>: <br> 166 * If the job generator is {@link FixedDomainConfigurationCountJobGenerator}, then this parameter represents the 167 * maximum number of domain configurations in a full harvest job. 168 */ 169 public static String JOBGEN_FIXED_CONFIG_COUNT_SNAPSHOT = "settings.harvester.scheduler.jobGen.config.fixedDomainCountSnapshot"; 170 171 /** 172 * <b>settings.harvester.scheduler.jobGen.config.excludeDomainsWithZeroBudget</b>: <br> 173 * If the job generator is {@link FixedDomainConfigurationCountJobGenerator}, then this parameter toggles whether or 174 * not domain configurations with a budget of zero (byte or objects) should be excluded from jobs. The default value 175 * is 'false'. 176 */ 177 public static String JOBGEN_FIXED_CONFIG_COUNT_EXCLUDE_ZERO_BUDGET = "settings.harvester.scheduler.jobGen.config.excludeDomainsWithZeroBudget"; 178 179 /** 180 * <b>settings.harvester.scheduler.jobGen.config.postponeUnregisteredChannel</b>: <br> 181 * If this property is true, then the job generator will postpone job generation for harvest definitions that are 182 * mapped to a harvest channel not registered to at least one harvester. The default value is 'true'. 183 */ 184 public static String JOBGEN_POSTPONE_UNREGISTERED_HARVEST_CHANNEL = "settings.harvester.scheduler.jobGen.config.postponeUnregisteredChannel"; 185 186 /** 187 * <b>settings.harvester.scheduler.jobGen.class</b>: <br> 188 * The fully qualified class name of the chosen job generator implementation, currently either 189 * {@link DefaultJobGenerator} or {@link FixedDomainConfigurationCountJobGenerator}. The default is 190 * {@link DefaultJobGenerator}. 191 */ 192 public static String JOBGEN_CLASS = "settings.harvester.scheduler.jobGen.class"; 193 194 /** 195 * <b>settings.harvester.scheduler.jobGen.config.splitByObjectLimit</b>: <br> 196 * By default the byte limit is used as the base criterion for how many domain configurations are put into one 197 * harvest job. However if this parameter is set to "true", then the object limit is used instead as the base 198 * criterion. 199 */ 200 public static String SPLIT_BY_OBJECTLIMIT = "settings.harvester.scheduler.jobGen.config.splitByObjectLimit"; 201 202 /** 203 * <b>settings.harvester.scheduler.jobGen.objectLimitIsSetByQuotaEnforcer</b>: <br> 204 * Controls whether the domain configuration object limit should be set in Heritrix's crawl order through the 205 * QuotaEnforcer configuration (parameter set to true) or through the frontier parameter 'queue-total-budget' ( 206 * parameter set to false). 207 * <p> 208 * Default value is true, as legacy implementation was to use only the QuotaEnforcer. 209 */ 210 public static String OBJECT_LIMIT_SET_BY_QUOTA_ENFORCER = "settings.harvester.scheduler.jobGen.objectLimitIsSetByQuotaEnforcer"; 211 212 /** 213 * <b>settings.harvester.scheduler.jobtimeouttime</b>:<br /> 214 * Time before a STARTED job times out and change status to FAILED. In seconds. 215 */ 216 public static String JOB_TIMEOUT_TIME = "settings.harvester.scheduler.jobtimeouttime"; 217 218 /** 219 * <b>settings.harvester.scheduler.jobgenerationperiode</b>: <br> 220 * The period between checking if new jobs should be generated, in seconds. This is one minute because that's the 221 * finest we can define in a harvest definition. 222 */ 223 public static String GENERATE_JOBS_PERIOD = "settings.harvester.scheduler.jobgenerationperiode"; 224 225 /** 226 * <b>settings.harvester.harvesting.serverDir</b>: <br> 227 * Each job gets a subdir of this dir. Job data is written and Heritrix writes to that subdir. 228 * TODO verify, if this is OK for Heritrix3 as well as Heritrix1 229 * NOTE THAT the jobsdir in Heritrix3 consists of ???????? 230 * 231 */ 232 public static String HARVEST_CONTROLLER_SERVERDIR = "settings.harvester.harvesting.serverDir"; 233 234 /** 235 * <b>settings.harvester.harvesting.minSpaceLeft</b>: <br> 236 * The minimum amount of free bytes in the serverDir required before accepting any harvest-jobs. 237 */ 238 public static String HARVEST_SERVERDIR_MINSPACE = "settings.harvester.harvesting.minSpaceLeft"; 239 240 /** 241 * <b>settings.harvester.harvesting.oldjobsDir</b>: <br> 242 * The directory in which data from old jobs is kept after uploading. Each directory from serverDir will be moved to 243 * here if any data remains, either due to failed uploads or because it wasn't attempted uploaded. 244 */ 245 public static String HARVEST_CONTROLLER_OLDJOBSDIR = "settings.harvester.harvesting.oldjobsDir"; 246 247 /** 248 * <b>settings.harvester.harvesting.channel</b>: <br> 249 * Harvest channel to take jobs from. This is the default channel assigned to the harvest controller. 250 * 251 * @see dk.netarkivet.harvester.datamodel.HarvestChannel <p> 252 * NOTE: this one is also used in SingleMBeanObject parsing information to System state 253 */ 254 public static String HARVEST_CONTROLLER_CHANNEL = "settings.harvester.harvesting.channel"; 255 256 /** 257 * <b>settings.harvester.harvesting.heritrix.inactivityTimeout</b>: <br> 258 * The timeout setting for aborting a crawl based on crawler-inactivity. If the crawler is inactive for this amount 259 * of seconds the crawl will be aborted. The inactivity is measured on the crawlController.activeToeCount(). 260 */ 261 public static String INACTIVITY_TIMEOUT_IN_SECS = "settings.harvester.harvesting.heritrix.inactivityTimeout"; 262 263 /** 264 * <b>settings.harvester.harvesting.heritrix.noresponseTimeout</b>: <br> 265 * The timeout value (in seconds) used in HeritrixLauncher for aborting crawl when no bytes are being received from 266 * web servers. 267 */ 268 public static String CRAWLER_TIMEOUT_NON_RESPONDING = "settings.harvester.harvesting.heritrix.noresponseTimeout"; 269 /** 270 * <b>settings.harvester.monitor.refreshInterval</b>:<br> 271 * Time interval in seconds after which the harvest monitor pages will be automatically refreshed. 272 */ 273 public static String HARVEST_MONITOR_REFRESH_INTERVAL = "settings.harvester.monitor.refreshInterval"; 274 275 /** 276 * <b>settings.harvester.monitor.historySampleRate</b>:<br> 277 * Time interval in seconds between historical records stores in the DB. Default value is 5 minutes. 278 */ 279 public static String HARVEST_MONITOR_HISTORY_SAMPLE_RATE = "settings.harvester.monitor.historySampleRate"; 280 281 /** 282 * <b>settings.harvester.monitor.historyChartGenIntervall</b>:<br> 283 * Time interval in seconds between regenerating the chart of historical data for a running job. Default value is 5 284 * minutes. 285 */ 286 public static String HARVEST_MONITOR_HISTORY_CHART_GEN_INTERVAL = "settings.harvester.monitor.historyChartGenInterval"; 287 288 /** 289 * <b>settings.harvester.monitor.displayedHistorySize</b>:<br> 290 * Maximum number of most recent history records displayed on the running job details page. 291 */ 292 public static String HARVEST_MONITOR_DISPLAYED_HISTORY_SIZE = "settings.harvester.monitor.displayedHistorySize"; 293 294 /** 295 * <b>settings.harvester.harvesting.heritrix.crawlLoopWaitTime</b>:<br> 296 * Time interval in seconds to wait during a crawl loop in the harvest controller. Default value is 20 seconds. 297 * 298 * TODO Maybe move this from the heritrix settings (settings.harvester.harvesting.heritrix) to 299 * settings.harvester.harvesting.controller. 300 * 301 */ 302 public static String CRAWL_LOOP_WAIT_TIME = "settings.harvester.harvesting.heritrix.crawlLoopWaitTime"; 303 304 /** 305 * <b>settings.harvester.harvesting.sendReadyInterval</b>:<br> 306 * Time interval in seconds to wait before transmitting a {@link HarvesterReadyMessage} to the {@link JobDispatcher} 307 * . 308 * <p> 309 * <p> 310 * Lower values will make the JobDispatcher detect ready harvester faster, but will make it more likely that the 311 * harvester may send two ready messages before a job is received, causing the JobDispatcher to dispatch two jobs. 312 * <p> 313 * Default value is 30 second. 314 */ 315 public static String SEND_READY_INTERVAL = "settings.harvester.harvesting.sendReadyInterval"; 316 317 /** 318 * <b>settings.harvester.harvesting.sendReadyDelay</b>:<br> 319 * Time in milliseconds to wait from starting to listen on the job queue to a potential ready message is sent to the 320 * HarvestJobManager. This small delay is used to retrieve any left over jobs on the queue before sending the ready 321 * message to the harvester. Default value is 1000 millisecond. 322 */ 323 public static String SEND_READY_DELAY = "settings.harvester.harvesting.sendReadyDelay"; 324 325 /** 326 * <b>settings.harvester.harvesting.frontier.frontierReportWaitTime</b>:<br> 327 * Time interval in seconds to wait between two requests to generate a full frontier report. Default value is 600 328 * seconds (10 min). 329 */ 330 public static String FRONTIER_REPORT_WAIT_TIME = "settings.harvester.harvesting.frontier.frontierReportWaitTime"; 331 332 /** 333 * <b>settings.harvester.harvesting.frontier.filter.class</b> Defines a filter to apply to the full frontier report. 334 * the default class: {@link TopTotalEnqueuesFilter} 335 */ 336 public static String FRONTIER_REPORT_FILTER_CLASS = "settings.harvester.harvesting.frontier.filter.class"; 337 338 /** 339 * <b>settings.harvester.harvesting.frontier.filter.args</b> Defines a frontier report filter's arguments. Arguments 340 * should be separated by semicolons. 341 */ 342 public static String FRONTIER_REPORT_FILTER_ARGS = "settings.harvester.harvesting.frontier.filter.args"; 343 344 /** 345 * <b>settings.harvester.harvesting.heritrix.abortIfConnectionLost</b>:<br> 346 * Boolean flag. If set to true, the harvest controller will abort the current crawl when the JMX connection is 347 * lost. If set to true it will only log a warning, leaving the crawl operator shutting down harvester manually. 348 * Default value is true. 349 * 350 * @see BnfHeritrixController 351 */ 352 public static String ABORT_IF_CONNECTION_LOST = "settings.harvester.harvesting.heritrix.abortIfConnectionLost"; 353 354 /** 355 * <b>settings.harvester.harvesting.heritrix.waitForReportGenerationTimeout</b>:<br> 356 * Maximum time in seconds to wait for Heritrix to generate report files once crawling is over. 357 */ 358 public static String WAIT_FOR_REPORT_GENERATION_TIMEOUT = "settings.harvester.harvesting.heritrix.waitForReportGenerationTimeout"; 359 360 /** 361 * <b>settings.harvester.harvesting.heritrix</b>: <br> 362 * The path to the Heritrix SETTINGS. 363 */ 364 public static String HERITRIX = "settings.harvester.harvesting.heritrix"; 365 366 /** 367 * <b>settings.harvester.harvesting.heritrix.adminName</b>: <br> 368 * The name used to access the Heritrix GUI. 369 */ 370 public static String HERITRIX_ADMIN_NAME = "settings.harvester.harvesting.heritrix.adminName"; 371 372 /** 373 * <b>settings.harvester.harvesting.heritrix.adminPassword</b>: <br> 374 * The password used to access the Heritrix GUI. 375 */ 376 public static String HERITRIX_ADMIN_PASSWORD = "settings.harvester.harvesting.heritrix.adminPassword"; 377 378 /** 379 * <b>settings.harvester.harvesting.heritrix.guiPort</b>: <br> 380 * Port used to access the Heritrix web user interface. This port must not be used by anything else on the machine. 381 * Note that apart from pausing a job, modifications done directly on Heritrix may cause unexpected breakage. 382 */ 383 public static String HERITRIX_GUI_PORT = "settings.harvester.harvesting.heritrix.guiPort"; 384 385 /** 386 * <b>settings.harvester.harvesting.heritrix.jmxPort</b>: <br> 387 * The port that Heritrix 1.14.4 uses to expose its JMX interface. This port must not be used by anything else on the 388 * machine, but does not need to be accessible from other machines unless you want to be able to use jconsole to 389 * access Heritrix directly. Note that apart from pausing a job, modifications done directly on Heritrix may cause 390 * unexpected breakage. Irrelevant for Heritrix 3+ 391 */ 392 public static String HERITRIX_JMX_PORT = "settings.harvester.harvesting.heritrix.jmxPort"; 393 394 /** 395 * <b>settings.harvester.harvesting.heritrix.jmxUsername</b>: <br> 396 * The username used to connect to Heritrix 1.14.4 JMX interface The username must correspond to the value stored in the 397 * jmxremote.password file (name defined in setting settings.common.jmx.passwordFile). 398 * Irrelevant for Heritrix 3+ 399 */ 400 public static String HERITRIX_JMX_USERNAME = "settings.harvester.harvesting.heritrix.jmxUsername"; 401 402 /** 403 * <b>settings.harvester.harvesting.heritrix.jmxPassword</b>: <br> 404 * The password used to connect to Heritrix JMX interface The password must correspond to the value stored in the 405 * jmxremote.password file (name defined in setting settings.common.jmx.passwordFile). 406 * Irrelevant for Heritrix 3+ 407 */ 408 public static String HERITRIX_JMX_PASSWORD = "settings.harvester.harvesting.heritrix.jmxPassword"; 409 410 /** 411 * <b>settings.harvester.harvesting.heritrix.heapSize</b>: <br> 412 * The heap size to use for the Heritrix sub-process. This should probably be fairly large. It can be specified in 413 * the same way as for the -Xmx argument to Java, e.g. 512M, 2G etc. 414 */ 415 public static String HERITRIX_HEAP_SIZE = "settings.harvester.harvesting.heritrix.heapSize"; 416 417 /** 418 * <b>settings.harvester.harvesting.heritrix.javaOpts</b>: <br> 419 * Additional JVM options for the Heritrix sub-process. By default there is no additional JVM option. 420 */ 421 public static String HERITRIX_JVM_OPTS = "settings.harvester.harvesting.heritrix.javaOpts"; 422 423 /** 424 * <b>settings.harvester.harvesting.heritrixControllerClass</b>:<br/> 425 * The implementation of the HeritrixController interface to be used. 426 */ 427 public static String HERITRIX_CONTROLLER_CLASS = "settings.harvester.harvesting.heritrixController.class"; 428 429 /** 430 * <b>settings.harvester.harvesting.heritrixLauncherClass</b>:<br/> 431 * The implementation of the HeritrixLauncher abstract class to be used. 432 */ 433 public static String HERITRIX_LAUNCHER_CLASS = "settings.harvester.harvesting.heritrixLauncher.class"; 434 435 /** 436 * <b>settings.harvester.harvesting.harvestReport</b>:<br/> 437 * The implementation of {@link HarvestReport} interface to be used. 438 */ 439 public static String HARVEST_REPORT_CLASS = "settings.harvester.harvesting.harvestReport.class"; 440 441 /** 442 * <b>settings.harvester.harvesting.harvestReport.disregardSeedsURLInfo</b>:<br/> 443 * Should we disregard seedURL-information and thus assign the harvested bytes to the domain of the harvested URL 444 * instead of the seed url domain? The default is false; 445 */ 446 public static String DISREGARD_SEEDURL_INFORMATION_IN_CRAWLLOG = "settings.harvester.harvesting.harvestReport.disregardSeedURLInfo"; 447 448 /** 449 * <b>settings.harvester.harvesting.deduplication.enabled</b>:<br/> 450 * This setting tells the system whether or not to use deduplication. This setting is true by default. 451 */ 452 public static String DEDUPLICATION_ENABLED = "settings.harvester.harvesting.deduplication.enabled"; 453 454 /** 455 * <b>settings.harvester.harvesting.metadata.heritrixFilePattern</b> This setting allows to filter which Heritrix 456 * files should be stored in the metadata (W)ARC file.. 457 * 458 * @see Pattern 459 */ 460 public static String METADATA_HERITRIX_FILE_PATTERN = "settings.harvester.harvesting.metadata.heritrixFilePattern"; 461 462 /** 463 * <b>settings.harvester.harvesting.metadata.reportFilePattern</b> This setting allows to filter which Heritrix 464 * files that should be stored in the metadata (W)ARC file are to be classified as a report. 465 * 466 * @see Pattern 467 */ 468 public static String METADATA_REPORT_FILE_PATTERN = "settings.harvester.harvesting.metadata.reportFilePattern"; 469 470 /** 471 * <b>settings.harvester.harvesting.metadata.logFilePattern</b> This setting allows to filter which Heritrix log 472 * files should be stored in the metadata (W)ARC file. 473 * 474 * @see Pattern 475 */ 476 public static String METADATA_LOG_FILE_PATTERN = "settings.harvester.harvesting.metadata.logFilePattern"; 477 478 /** 479 * <b>settings.harvester.harvesting.metadata.generateArchiveFilesReport</b> This setting is a boolean flag that 480 * enables/disables the generation of an ARC/WARC files report. Default value is 'true'. 481 * 482 * @see HarvestDocumentation#documentHarvest(dk.netarkivet.harvester.harvesting.IngestableFiles) 483 */ 484 public static String METADATA_GENERATE_ARCHIVE_FILES_REPORT = "settings.harvester.harvesting.metadata.archiveFilesReport.generate"; 485 486 /** 487 * <b>settings.harvester.harvesting.metadata.archiveFilesReportName</b> If 488 * {@link #METADATA_GENERATE_ARCHIVE_FILES_REPORT} is set to true, sets the name of the generated report file. 489 * Default value is 'archivefiles-report.txt'. 490 * 491 * FIXME: not easily portable to H3, as it depends on information in heritrix_out.log no longer available. 492 * 493 * @see HarvestDocumentation#documentHarvest(dk.netarkivet.harvester.harvesting.IngestableFiles) 494 */ 495 public static String METADATA_ARCHIVE_FILES_REPORT_NAME = "settings.harvester.harvesting.metadata.archiveFilesReport.fileName"; 496 497 /** 498 * <b>settings.harvester.harvesting.metadata.archiveFilesReportName</b> If 499 * {@link #METADATA_GENERATE_ARCHIVE_FILES_REPORT} is set to true, sets the header of the generated report file. 500 * This setting should generally be left to its default value, which is '[ARCHIVEFILE] [Opened] [Closed] [Size]'. 501 * 502 * @see HarvestDocumentation#documentHarvest(dk.netarkivet.harvester.harvesting.IngestableFiles) 503 */ 504 public static String METADATA_ARCHIVE_FILES_REPORT_HEADER = "settings.harvester.harvesting.metadata.archiveFilesReport.fileHeader"; 505 506 /** 507 * <b>settings.harvester.aliases.timeout</b> The amount of time in seconds before an alias times out, and needs to 508 * be re-evaluated. The default value is one year, i.e 31536000 seconds. 509 */ 510 public static String ALIAS_TIMEOUT = "settings.harvester.aliases.timeout"; 511 512 /** 513 * <b>settings.harvester.harvesting.continuationFromHeritrixRecoverlogEnabled</b>:</br> Setting for whether or not a 514 * restarted job should try fetching the recoverlog of the previous failed job, and ask Heritrix to continue from 515 * this log. The default is false. 516 */ 517 public static String RECOVERlOG_CONTINUATION_ENABLED = "settings.harvester.harvesting.continuationFromHeritrixRecoverlogEnabled"; 518 519 /** 520 * <b>settings.harvester.harvesting.metadata.metadataFormat</b> The dataformat used by Netarchivesuite to write the 521 * metadata associated with a given harvest job. default: arc (alternative: warc) 522 */ 523 public static String METADATA_FORMAT = "settings.harvester.harvesting.metadata.metadataFormat"; 524 525 /** 526 * <b>settings.harvester.harvesting.heritrix.archiveFormat</b> The dataformat used by heritrix to write the 527 * harvested data. default: warc (alternative: arc) 528 */ 529 public static String HERITRIX_ARCHIVE_FORMAT = "settings.harvester.harvesting.heritrix.archiveFormat"; 530 /** 531 * <b>settings.harvester.harvesting.heritrix.archiveNaming.class</b> The class implementing the chosen way of naming 532 * your archive-files default: LegacyNamingConvention. This class decides what to put into the Heritrix "prefix" 533 * property of the org.archive.crawler.writer.ARCWriterProcessor and/or 534 * org.archive.crawler.writer.WARCWriterProcessor. 535 */ 536 public static String HERITRIX_ARCHIVE_NAMING_CLASS = "settings.harvester.harvesting.heritrix.archiveNaming.class"; 537 538 /** 539 * <b>settings.harvester.harvesting.heritrix.warc.skipIdenticalDigests</b> Represents the 'skip-identical-digests' 540 * setting in the Heritrix WARCWriterProcessor. The default is false. 541 */ 542 public static String HERITRIX_WARC_SKIP_IDENTICAL_DIGESTS = "settings.harvester.harvesting.heritrix.warc.skipIdenticalDigests"; 543 /** 544 * <b>settings.harvester.harvesting.heritrix.warc.writeRequests</b> Represents the 'write-requests' setting in the 545 * Heritrix WARCWriterProcessor. The default is true 546 */ 547 public static String HERITRIX_WARC_WRITE_REQUESTS = "settings.harvester.harvesting.heritrix.warc.writeRequests"; 548 /** 549 * <b>settings.harvester.harvesting.heritrix.warc.writeMetadata</b> Represents the 'write-metadata' setting in the 550 * Heritrix WARCWriterProcessor. The default is true. 551 */ 552 public static String HERITRIX_WARC_WRITE_METADATA = "settings.harvester.harvesting.heritrix.warc.writeMetadata"; 553 /** 554 * <b>settings.harvester.harvesting.heritrix.warc.writeRevisitForIdenticalDigests</b> Represents the 555 * 'write-revisit-for-identical-digests' setting in the Heritrix WARCWriterProcessor. The default is true. 556 */ 557 public static String HERITRIX_WARC_WRITE_REVISIT_FOR_IDENTICAL_DIGESTS = "settings.harvester.harvesting.heritrix.warc.writeRevisitForIdenticalDigests"; 558 /** 559 * <b>settings.harvester.harvesting.heritrix.warc.writeRevisitForNotModified</b> Represents the 560 * 'write-revisit-for-not-modified' setting in the Heritrix WARCWriterProcessor. The default is true. 561 */ 562 public static String HERITRIX_WARC_WRITE_REVISIT_FOR_NOT_MODIFIED = "settings.harvester.harvesting.heritrix.warc.writeRevisitForNotModified"; 563 564 /** 565 * <b>settings.harvester.harvesting.heritrix.warc.startNewFilesOnCheckpoint</b> Represents the 566 * 'startNewFilesOnCheckpoint' setting in the Heritrix WARCWriterProcessor. Only available with H3. The default is true. 567 */ 568 public static String HERITRIX_WARC_START_NEW_FILES_ON_CHECKPOINT 569 = "settings.harvester.harvesting.heritrix.warc.startNewFilesOnCheckpoint"; 570 571 572 573 /** 574 * <b>settings.harvester.harvesting.heritrix.version</b> Represents the version of Heritrix used by Netarchivesuite 575 * The default is h3. The optional value is h1. 576 * 577 * If h1 is chosen, we assume that our templates is h1, as well. 578 * If h3 is chosen, we assume that our templates is h3, as well. 579 * There is no attempt at migration from one to the other. This must be done by an commandline-tool. 580 */ 581 public static String HERITRIX_VERSION = "settings.harvester.harvesting.heritrix.version"; 582 /** 583 * <b>settings.harvester.harvesting.heritrix.bundle</b>Points to the Heritrix3 zipfile bundled with 584 * netarchiveSuite classes. Currently no default value 585 */ 586 public static String HERITRIX3_BUNDLE = "settings.harvester.harvesting.heritrix.bundle"; 587 588 /** 589 * <b>settings.harvester.harvesting.heritrix.certificate</b>Points to the jks keystore to use for connection to the 590 * Heritrix3 rest api. If undefined the keystore provided with the heritrix3 bundler is used. 591 */ 592 public static String HERITRIX3_CERTIFICATE = "settings.harvester.harvesting.heritrix.certificate"; 593 594 public static String HERITRIX3_CERTIFICATE_PASSWORD = "settings.harvester.harvesting.heritrix.certificatePassword"; 595 596 /** 597 * <b>settings.harvester.performer</b>: <br> 598 * The agent performing these harvests. The default is: "" 599 */ 600 public static String PERFORMER = "settings.harvester.performer"; 601 602 /***************************/ 603 /* Indexserver - settings. */ 604 /***************************/ 605 606 /** 607 * <b>settings.harvester.indexserver.requestdir</b>: <br> 608 * Setting for where the requests of the indexserver are stored. 609 */ 610 public static String INDEXSERVER_INDEXING_REQUESTDIR = "settings.harvester.indexserver.requestdir"; 611 612 /** 613 * <b>settings.harvester.indexserver.maxclients</b>: <br> 614 * Setting for the max number of clients the indexserver can handle simultaneously. 615 */ 616 public static String INDEXSERVER_INDEXING_MAXCLIENTS = "settings.harvester.indexserver.maxclients"; 617 618 /** 619 * <b>settings.harvester.indexserver.maxthreads</b>: <br> 620 * Setting for the max number of threads the deduplication indexer shall use. 621 */ 622 public static String INDEXSERVER_INDEXING_MAXTHREADS = "settings.harvester.indexserver.maxthreads"; 623 /** 624 * <b>settings.harvester.indexserver.checkinterval</b>: <br> 625 * Setting for the time in milliseconds between each check of the state of sub-indexing. Default: 30 seconds (30000 626 * milliseconds). 627 */ 628 public static String INDEXSERVER_INDEXING_CHECKINTERVAL = "settings.harvester.indexserver.checkinterval"; 629 630 /** 631 * <b>settings.harvester.indexserver.indexingtimeout</b>: <br> 632 * Setting for the indexing timeout in milliseconds. The default is 259200000 (3 days). 633 */ 634 public static String INDEXSERVER_INDEXING_TIMEOUT = "settings.harvester.indexserver.indexingtimeout"; 635 636 /** 637 * <b>settings.harvester.indexserver.maxsegments</b>: <br> 638 * Setting for how many segments we will accept in our lucene indices. The default is 15. 639 */ 640 public static String INDEXSERVER_INDEXING_MAX_SEGMENTS = "settings.harvester.indexserver.maxsegments"; 641 642 /** 643 * <b>settings.harvester.indexserver.listeningcheckinterval</b>: <br> 644 * Setting for the interval between each listening check in milliseconds. The default is 30000 (5 minutes). 645 */ 646 public static String INDEXSERVER_INDEXING_LISTENING_INTERVAL = "settings.harvester.indexserver.listeningcheckinterval"; 647 /** 648 * <b>settings.archive.indexserver.satisfactorythresholdpercentage</b>: <br> 649 * Setting for the satisfactory threshold of the indexing result as a percentage. The default is 70 percent 650 */ 651 public static String INDEXSERVER_INDEXING_SATISFACTORYTHRESHOLD_PERCENTAGE = "settings.harvester.indexserver.satisfactorythresholdpercentage"; 652 653 /** 654 * <b>settings.harvester.indexserver.indexrequestserver.class</b>: <br> 655 * Setting for which type of indexrequestserver to use. The default is: 656 * {@link dk.netarkivet.harvester.indexserver.distribute.IndexRequestServer} 657 */ 658 public static String INDEXREQUEST_SERVER_CLASS = "settings.harvester.indexserver.indexrequestserver.class"; 659 660 /** 661 * b>settings.harvester.indexserver.lookfordataInAllBitarchiveReplicas</b>: <br> 662 * Setting for whether or not data not found in the default bitarchive replica shall be looked for in other 663 * bitarchive replicas. The default is false. 664 */ 665 public static String INDEXSERVER_INDEXING_LOOKFORDATAINOTHERBITARCHIVEREPLICAS = "settings.harvester.indexserver.lookfordataInAllBitarchiveReplicas"; 666 667 /***************************/ 668 /* Viewerproxy - settings. */ 669 /***************************/ 670 671 /** 672 * <b>settings.viewerproxy.baseDir</b>: <br> 673 * The main directory for the ViewerProxy, used for storing the Lucene index for the jobs being viewed. This 674 * directory can be used by multiple ViewerProxy applications running on the same machine. 675 */ 676 public static String VIEWERPROXY_DIR = "settings.harvester.viewerproxy.baseDir"; 677 678 /** 679 * <b>settings.viewerproxy.tryLookupUriAsFtp</b>: <br> 680 * If we fail to lookup an URI, we will try changing the protocol to ftp, if this setting is set to true. The 681 * default is false. 682 */ 683 public static String TRY_LOOKUP_URI_AS_FTP = "settings.harvester.viewerproxy.tryLookupUriAsFtp"; 684 685 /** 686 * <b>settings.viewerproxy.maxSizeInBrowser</b> The size (in bytes) of the largest object to be returned for viewing 687 * in the browser window. Larger objects will be returned with the appropriate http header for saving them to a 688 * file. 689 */ 690 public static String MAXIMUM_OBJECT_IN_BROWSER = "settings.harvester.viewerproxy.maxSizeInBrowser"; 691 692 /** 693 * The maximum length (in lines) of crawllog to be displayed in a browser window. 694 */ 695 public static String MAX_CRAWLLOG_IN_BROWSER = "settings.harvester.webinterface.maxCrawlLogInBrowser"; 696 697}