001/* Copyright (C) 2003 Internet Archive. 002 * 003 * This file is part of the Heritrix web crawler (crawler.archive.org). 004 * 005 * Heritrix is free software; you can redistribute it and/or modify 006 * it under the terms of the GNU Lesser Public License as published by 007 * the Free Software Foundation; either version 2.1 of the License, or 008 * any later version. 009 * 010 * Heritrix is distributed in the hope that it will be useful, 011 * but WITHOUT ANY WARRANTY; without even the implied warranty of 012 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 013 * GNU Lesser Public License for more details. 014 * 015 * You should have received a copy of the GNU Lesser Public License 016 * along with Heritrix; if not, write to the Free Software 017 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA 018 * 019 * Created on Nov 17, 2003 020 * 021 */ 022package dk.netarkivet.harvester.harvesting.extractor; 023 024import java.io.IOException; 025import java.util.logging.Logger; 026import java.util.regex.Matcher; 027 028import org.apache.commons.httpclient.URIException; 029import org.apache.commons.lang.StringEscapeUtils; 030import org.archive.crawler.datamodel.CoreAttributeConstants; 031import org.archive.crawler.datamodel.CrawlURI; 032import org.archive.crawler.extractor.Extractor; 033import org.archive.crawler.extractor.Link; 034import org.archive.crawler.framework.CrawlController; 035import org.archive.io.ReplayCharSequence; 036import org.archive.net.UURI; 037import org.archive.util.DevUtils; 038import org.archive.util.TextUtils; 039import org.archive.util.UriUtils; 040 041/** 042 * Processes Javascript files for strings that are likely to be crawlable URIs. 043 * <p> 044 * contributor gojomo contributor szznax contributor svc 045 */ 046public class ExtractorJS extends Extractor implements CoreAttributeConstants { 047 048 private static final long serialVersionUID = -2231962381454717720L; 049 050 private static final Logger LOGGER = Logger.getLogger(ExtractorJS.class.getName()); 051 052 // finds whitespace-free strings in Javascript 053 // (areas between paired ' or " characters, possibly backslash-quoted 054 // on the ends, but not in the middle) 055 static final String JAVASCRIPT_STRING_EXTRACTOR = "(\\\\{0,8}+(?:\"|\'))(\\S{0," + UURI.MAX_URL_LENGTH 056 + "}?)(?:\\1)"; 057 // GROUPS: 058 // (G1) ' or " with optional leading backslashes 059 // (G2) whitespace-free string delimited on boths ends by G1 060 061 protected long numberOfCURIsHandled = 0; 062 protected static long numberOfLinksExtracted = 0; 063 064 // URIs known to produce false-positives with the current JS extractor. 065 // e.g. currently (2.0.3) the JS extractor produces 13 false-positive 066 // URIs from http://www.google-analytics.com/urchin.js and only 2 067 // good URIs, which are merely one pixel images. 068 // TODO: remove this blacklist when JS extractor is improved 069 protected final static String[] EXTRACTOR_URI_EXCEPTIONS = {"http://www.google-analytics.com/urchin.js"}; 070 071 /** 072 * @param name 073 */ 074 public ExtractorJS(String name) { 075 super(name, "JavaScript extractor. Link extraction on JavaScript" + " files (.js)."); 076 } 077 078 /* 079 * (non-Javadoc) 080 * 081 * @see org.archive.crawler.framework.Processor#process(org.archive.crawler.datamodel.CrawlURI) 082 */ 083 public void extract(CrawlURI curi) { 084 // special-cases, for when we know our current JS extractor does poorly. 085 // TODO: remove this test when JS extractor is improved 086 for (String s : EXTRACTOR_URI_EXCEPTIONS) { 087 if (curi.toString().equals(s)) { 088 return; 089 } 090 } 091 092 if (!isHttpTransactionContentToProcess(curi)) { 093 return; 094 } 095 String contentType = curi.getContentType(); 096 if ((contentType == null)) { 097 return; 098 } 099 // If content type is not js and if the viaContext 100 // does not begin with 'script', return. 101 if ((contentType.indexOf("javascript") < 0) 102 && (contentType.indexOf("jscript") < 0) 103 && (contentType.indexOf("ecmascript") < 0) 104 && (!curi.toString().toLowerCase().endsWith(".js")) 105 && (curi.getViaContext() == null || !curi.getViaContext().toString().toLowerCase().startsWith("script"))) { 106 return; 107 } 108 109 this.numberOfCURIsHandled++; 110 111 ReplayCharSequence cs = null; 112 try { 113 cs = curi.getHttpRecorder().getReplayCharSequence(); 114 } catch (IOException e) { 115 curi.addLocalizedError(this.getName(), e, "Failed get of replay char sequence."); 116 } 117 if (cs == null) { 118 LOGGER.warning("Failed getting ReplayCharSequence: " + curi.toString()); 119 return; 120 } 121 122 try { 123 try { 124 numberOfLinksExtracted += considerStrings(curi, cs, getController(), true); 125 } catch (StackOverflowError e) { 126 DevUtils.warnHandle(e, "ExtractorJS StackOverflowError"); 127 } 128 // Set flag to indicate that link extraction is completed. 129 curi.linkExtractorFinished(); 130 } finally { 131 // Done w/ the ReplayCharSequence. Close it. 132 if (cs != null) { 133 try { 134 cs.close(); 135 } catch (IOException ioe) { 136 LOGGER.warning(TextUtils.exceptionToString("Failed close of ReplayCharSequence.", ioe)); 137 } 138 } 139 } 140 } 141 142 public static long considerStrings(CrawlURI curi, CharSequence cs, CrawlController controller, 143 boolean handlingJSFile) { 144 long foundLinks = 0; 145 Matcher strings = TextUtils.getMatcher(JAVASCRIPT_STRING_EXTRACTOR, cs); 146 while (strings.find()) { 147 CharSequence subsequence = cs.subSequence(strings.start(2), strings.end(2)); 148 149 if (UriUtils.isLikelyUriJavascriptContextLegacy(subsequence)) { 150 String string = subsequence.toString(); 151 string = StringEscapeUtils.unescapeJavaScript(string); 152 string = UriUtils.speculativeFixup(string, curi.getUURI()); 153 foundLinks++; 154 try { 155 if (handlingJSFile) { 156 curi.createAndAddLinkRelativeToVia(string, Link.JS_MISC, Link.SPECULATIVE_HOP); 157 } else { 158 curi.createAndAddLinkRelativeToBase(string, Link.JS_MISC, Link.SPECULATIVE_HOP); 159 } 160 } catch (URIException e) { 161 // There may not be a controller (e.g. If we're being run 162 // by the extractor tool). 163 if (controller != null) { 164 controller.logUriError(e, curi.getUURI(), string); 165 } else { 166 LOGGER.info(curi + ", " + string + ": " + e.getMessage()); 167 } 168 } 169 } else { 170 foundLinks += considerStrings(curi, subsequence, controller, handlingJSFile); 171 } 172 } 173 TextUtils.recycleMatcher(strings); 174 return foundLinks; 175 } 176 177 /* 178 * (non-Javadoc) 179 * 180 * @see org.archive.crawler.framework.Processor#report() 181 */ 182 public String report() { 183 StringBuffer ret = new StringBuffer(); 184 ret.append("Processor: dk.netarkivet.harvester.harvesting.extractor.ExtractorJS\n"); 185 ret.append(" Function: Link extraction on JavaScript code\n"); 186 ret.append(" CrawlURIs handled: " + numberOfCURIsHandled + "\n"); 187 ret.append(" Links extracted: " + numberOfLinksExtracted + "\n\n"); 188 189 return ret.toString(); 190 } 191}