Source code

001/*
002 * #%L
003 * Netarchivesuite - heritrix 3 monitor
004 * %%
005 * Copyright (C) 2005 - 2018 The Royal Danish Library, 
006 *             the National Library of France and the Austrian National Library.
007 * %%
008 * This program is free software: you can redistribute it and/or modify
009 * it under the terms of the GNU Lesser General Public License as
010 * published by the Free Software Foundation, either version 2.1 of the
011 * License, or (at your option) any later version.
012 * 
013 * This program is distributed in the hope that it will be useful,
014 * but WITHOUT ANY WARRANTY; without even the implied warranty of
015 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
016 * GNU General Lesser Public License for more details.
017 * 
018 * You should have received a copy of the GNU General Lesser Public
019 * License along with this program.  If not, see
020 * <http://www.gnu.org/licenses/lgpl-2.1.html>.
021 * #L%
022 */
023
024package dk.netarkivet.heritrix3.monitor.resources;
025
026import java.io.IOException;
027import java.net.URLEncoder;
028import java.util.List;
029import java.util.Locale;
030import java.util.regex.Matcher;
031import java.util.regex.Pattern;
032
033import javax.servlet.ServletContext;
034import javax.servlet.ServletOutputStream;
035import javax.servlet.http.HttpServletRequest;
036import javax.servlet.http.HttpServletResponse;
037
038import org.apache.commons.lang.StringEscapeUtils;
039import org.netarchivesuite.heritrix3wrapper.ScriptResult;
040
041import com.antiaction.common.filter.Caching;
042import com.antiaction.common.templateengine.TemplateBuilderFactory;
043
044import dk.netarkivet.heritrix3.monitor.Heritrix3JobMonitor;
045import dk.netarkivet.heritrix3.monitor.NASEnvironment;
046import dk.netarkivet.heritrix3.monitor.NASUser;
047import dk.netarkivet.heritrix3.monitor.Pagination;
048import dk.netarkivet.heritrix3.monitor.ResourceAbstract;
049import dk.netarkivet.heritrix3.monitor.ResourceManagerAbstract;
050import dk.netarkivet.heritrix3.monitor.HttpLocaleHandler.HttpLocale;
051
052public class H3FrontierResource implements ResourceAbstract {
053
054    private NASEnvironment environment;
055
056    protected int R_FRONTIER = -1;
057    
058    @Override
059    public void resources_init(NASEnvironment environment) {
060        this.environment = environment;
061    }
062
063    @Override
064    public void resources_add(ResourceManagerAbstract resourceManager) {
065        R_FRONTIER = resourceManager.resource_add(this, "/job/<numeric>/frontier/", false);
066    }
067
068    @Override
069    public void resource_service(ServletContext servletContext, NASUser nas_user, HttpServletRequest req, HttpServletResponse resp, HttpLocale httpLocale, int resource_id, List<Integer> numerics, String pathInfo) throws IOException {
070        if (NASEnvironment.contextPath == null) {
071            NASEnvironment.contextPath = req.getContextPath();
072        }
073        if (NASEnvironment.servicePath == null) {
074            NASEnvironment.servicePath = req.getContextPath() + req.getServletPath() + "/";
075        }
076        String method = req.getMethod().toUpperCase();
077        if (resource_id == R_FRONTIER) {
078            if ("GET".equals(method) || "POST".equals(method)) {
079                frontier_list(req, resp, httpLocale, numerics);
080            }
081        }
082    }
083
084    public void frontier_list(HttpServletRequest req, HttpServletResponse resp, HttpLocale httpLocale, List<Integer> numerics) throws IOException {
085        Locale locale = httpLocale.locale;
086        resp.setContentType("text/html; charset=UTF-8");
087        ServletOutputStream out = resp.getOutputStream();
088        Caching.caching_disable_headers(resp);
089
090        TemplateBuilderFactory<MasterTemplateBuilder> masterTplBuilderFactory = TemplateBuilderFactory.getInstance(environment.templateMaster, "master.tpl", "UTF-8", MasterTemplateBuilder.class);
091        MasterTemplateBuilder masterTplBuilder = masterTplBuilderFactory.getTemplateBuilder();
092
093        StringBuilder sb = new StringBuilder();
094
095        long lines;
096        long linesPerPage = 100;
097        long page = 1;
098        long pages = 0;
099        String q = null;
100
101        String tmpStr;
102        tmpStr = req.getParameter("page");
103        if (tmpStr != null && tmpStr.length() > 0) {
104            try {
105                page = Long.parseLong(tmpStr);
106            } catch (NumberFormatException e) {
107            }
108        }
109        tmpStr = req.getParameter("itemsperpage");
110        if (tmpStr != null && tmpStr.length() > 0) {
111            try {
112                linesPerPage = Long.parseLong(tmpStr);
113            } catch (NumberFormatException e) {
114            }
115        }
116
117        if (linesPerPage < 25) {
118            linesPerPage = 25;
119        }
120
121        String initials = req.getParameter("initials");
122        if (initials == null) {
123            initials = "";
124        }
125
126        tmpStr = req.getParameter("q");
127        if (tmpStr != null && tmpStr.length() > 0 && !tmpStr.equalsIgnoreCase(".*")) {
128            q = tmpStr;
129        } else {
130                q = "";
131        }
132
133        String additionalParams = "";
134        if (q.length() > 0) {
135            additionalParams += "&q=" + URLEncoder.encode(q, "UTF-8");
136        }
137        if (initials.length() > 0) {
138            additionalParams += "&initials=" + URLEncoder.encode(initials, "UTF-8");
139        }
140
141        String script = environment.NAS_GROOVY_SCRIPT;
142
143        String deleteStr = req.getParameter("delete");
144        if (deleteStr != null && "1".equals(deleteStr) && initials != null && initials.length() > 0) {
145            script += "\n";
146            script += "\ninitials = \"" + initials + "\"";
147            script += "\ndeleteFromFrontier '" + q + "'\n";
148        } else {
149            script += "\n";
150            script += "\nlistFrontier '" + q + "', " + linesPerPage + ", " + (page - 1) + "\n";
151        }
152
153        long jobId = numerics.get(0);
154        Heritrix3JobMonitor h3Job = environment.h3JobMonitorThread.getRunningH3Job(jobId);
155
156        if (h3Job != null && h3Job.isReady()) {
157            if (deleteStr != null && "1".equals(deleteStr) && (initials == null || initials.length() == 0)) {
158                //sb.append("<span style=\"text-color: red;\">Initials required to delete from the frontier queue!</span><br />\n");
159                sb.append("<div class=\"notify notify-red\"><span class=\"symbol icon-error\"></span> Initials required to delete from the frontier queue!</div>");
160            }
161
162            sb.append("<form class=\"form-horizontal\" action=\"?\" name=\"insert_form\" method=\"post\" enctype=\"application/x-www-form-urlencoded\" accept-charset=\"utf-8\">\n");
163            sb.append("<label for=\"limit\">Lines per page:</label>");
164            sb.append("<input type=\"text\" id=\"itemsperpage\" name=\"itemsperpage\" value=\"" + linesPerPage + "\" placeholder=\"return limit\">\n");
165            sb.append("<label for=\"q\">Filter regex:</label>");
166            sb.append("<input type=\"text\" id=\"q\" name=\"q\" value=\"" + q + "\" placeholder=\"regex\" style=\"display:inline;width:350px;\">\n");
167            sb.append("<button type=\"submit\" name=\"show\" value=\"1\" class=\"btn btn-success\"><i class=\"icon-white icon-thumbs-up\"></i> Show</button>\n");
168            sb.append("&nbsp;");
169            sb.append("<label for=\"initials\">User initials:</label>");
170            sb.append("<input type=\"text\" id=\"initials\" name=\"initials\" value=\"" + initials  + "\" placeholder=\"initials\">\n");
171            sb.append("<button type=\"submit\" name=\"delete\" value=\"1\" class=\"btn btn-danger\"><i class=\"icon-white icon-trash\"></i> Delete</button>\n");
172            sb.append("</form>\n");
173
174            ScriptResult scriptResult = h3Job.h3wrapper.ExecuteShellScriptInJob(h3Job.jobResult.job.shortName, "groovy", script);
175            lines = extractLinesAmount(scriptResult);
176
177            if (lines > 0) {
178                pages = (lines + linesPerPage - 1) / linesPerPage;
179                if (pages == 0) {
180                    pages = 1;
181                }
182            }
183
184            if (page > pages) {
185                page = pages;
186            }
187
188            if (scriptResult != null && scriptResult.script != null) {
189                if (scriptResult.script.failure) {
190                        if (scriptResult.script.stackTrace != null) {
191                        sb.append("<h5>Script failed with the following stacktrace:</h5>\n");
192                        sb.append("<pre>\n");
193                        sb.append(StringEscapeUtils.escapeHtml(scriptResult.script.stackTrace));
194                        sb.append("</pre>\n");
195                        } else if (scriptResult.script.exception != null) {
196                        sb.append("<h5>Script failed with the following message:</h5>\n");
197                        sb.append("<pre>\n");
198                        sb.append(StringEscapeUtils.escapeHtml(scriptResult.script.exception));
199                        sb.append("</pre>\n");
200                        } else {
201                        sb.append("<b>Unknown script failure!</b></br>\n");
202                        }
203                        sb.append("<h5>Raw script result Xml:</h5>\n");
204                    sb.append("<pre>");
205                    sb.append(StringEscapeUtils.escapeHtml(new String(scriptResult.response, "UTF-8")));
206                    sb.append("</pre>");
207                } else {
208                    sb.append("<div style=\"float:left;margin: 20px 0px;\">\n");
209                    sb.append("<span>Matching lines: ");
210                    sb.append(lines);
211                    sb.append(" URIs</span>\n");
212                    sb.append("</div>\n");
213                    sb.append(Pagination.getPagination(page, linesPerPage, pages, false, additionalParams));
214                    sb.append("<div style=\"clear:both;\"></div>");
215                    sb.append("<div>\n");
216                    sb.append("<pre>\n");
217                    //System.out.println(new String(scriptResult.response, "UTF-8"));
218                    if (scriptResult != null && scriptResult.script != null) {
219                        if (scriptResult.script.htmlOutput != null) {
220                            sb.append("<fieldset><!--<legend>htmlOut</legend>-->");
221                            sb.append(scriptResult.script.htmlOutput);
222                            sb.append("</fieldset><br />\n");
223                        }
224                        if (scriptResult.script.rawOutput != null) {
225                            sb.append("<fieldset><!--<legend>rawOut</legend>-->");
226                            sb.append("<pre>");
227                            sb.append(scriptResult.script.rawOutput);
228                            sb.append("</pre>");
229                            sb.append("</fieldset><br />\n");
230                        }
231                    }
232                    sb.append("</pre>\n");
233                    sb.append("</div>\n");
234                    sb.append(Pagination.getPagination(page, linesPerPage, pages, false, additionalParams));
235                    sb.append("</form>");
236                }
237            } else {
238                sb.append("<b>Script did not return any response!</b><br/>\n");
239            }
240        } else {
241            sb.append("Job ");
242            sb.append(jobId);
243            sb.append(" is not running.");
244        }
245
246        StringBuilder menuSb = masterTplBuilder.buildMenu(new StringBuilder(), req, locale, h3Job);
247
248        masterTplBuilder.insertContent("Job " + jobId + " Frontier", menuSb.toString(), httpLocale.generateLanguageLinks(),
249                        "Job " + jobId + " Frontier", sb.toString(), "").write(out);
250
251        out.flush();
252        out.close();
253    }
254    
255    private Long extractLinesAmount(ScriptResult scriptResult) {
256        Pattern pattern = Pattern.compile("\\d+");
257        if (scriptResult != null && scriptResult.script != null) {
258            try {
259                if (scriptResult.script.htmlOutput != null) {
260                    Matcher matcher = pattern.matcher(scriptResult.script.htmlOutput);
261                    matcher.find();
262                    String str = scriptResult.script.htmlOutput.substring(matcher.start(), matcher.end());
263                    scriptResult.script.htmlOutput = scriptResult.script.htmlOutput.substring(matcher.end());
264                    return Long.parseLong(str);
265                } else {
266                    if (scriptResult.script.rawOutput != null) {
267                        Matcher matcher = pattern.matcher(scriptResult.script.rawOutput);
268                        matcher.find();
269                        String str = scriptResult.script.rawOutput.substring(matcher.start(), matcher.end());
270                        scriptResult.script.rawOutput = scriptResult.script.rawOutput.substring(matcher.end());
271                        return Long.parseLong(str);
272                    }
273                }
274            }
275            catch (Exception ex) {
276                return 1L;
277            }
278        }
279        return 1L;
280    }
281
282}