Skip to content

Commit

Permalink
Merge pull request #126 from HHS/develop
Browse files Browse the repository at this point in the history
Revise logging to use a JSON log and add version
  • Loading branch information
danizen committed Mar 11, 2019
2 parents c06c290 + c606e40 commit 94a793e
Show file tree
Hide file tree
Showing 18 changed files with 247 additions and 32 deletions.
2 changes: 1 addition & 1 deletion pom.xml
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@
<groupId>gov.nih.nlm.lode</groupId>
<artifactId>meshrdf</artifactId>
<packaging>pom</packaging>
<version>1.0.7</version>
<version>1.0.8</version>
<name>NLM MeSH RDF parent project</name>
<scm>
<url>https://github.com/HHS/meshrdf</url>
Expand Down
13 changes: 12 additions & 1 deletion webui/pom.xml
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@
<parent>
<groupId>gov.nih.nlm.lode</groupId>
<artifactId>meshrdf</artifactId>
<version>1.0.7</version>
<version>1.0.8</version>
</parent>

<properties>
Expand Down Expand Up @@ -61,6 +61,10 @@
<include>*.owl</include>
</includes>
</resource>
<resource>
<directory>src/main/about</directory>
<filtering>true</filtering>
</resource>
</webResources>
<overlays>
<overlay>
Expand Down Expand Up @@ -206,6 +210,13 @@
<scope>test</scope>
</dependency>

<!-- Enable JSON Logging -->
<dependency>
<groupId>net.logstash.log4j</groupId>
<artifactId>jsonevent-layout</artifactId>
<version>1.7</version>
</dependency>

<!--
Build and test requires the servlet and JSP API.
We use the version for Tomcat 7, even though we
Expand Down
1 change: 1 addition & 0 deletions webui/src/main/about/internal/about.html
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
${project.version}
Original file line number Diff line number Diff line change
@@ -0,0 +1,45 @@
package gov.nih.nlm.lode.servlet;

import java.io.IOException;

import javax.servlet.ServletOutputStream;
import javax.servlet.WriteListener;


public class CountingServletOutputStream extends ServletOutputStream {
private ServletOutputStream originalStream;
private int count = 0;

public CountingServletOutputStream(ServletOutputStream wrappedStream) {
this.originalStream = wrappedStream;
}

@Override
public boolean isReady() {
return originalStream.isReady();
}

@Override
public void setWriteListener(WriteListener writeListener) {
originalStream.setWriteListener(writeListener);
}

@Override
public void write(int b) throws IOException {
count += 1;
originalStream.write(b);
}

@Override
public void close() throws IOException {
originalStream.close();
}

public int getCount() {
return count;
}

public void resetCount() {
count = 0;
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,41 @@
package gov.nih.nlm.lode.servlet;

import java.io.IOException;
import java.io.PrintWriter;

import javax.servlet.ServletOutputStream;
import javax.servlet.http.HttpServletResponse;
import javax.servlet.http.HttpServletResponseWrapper;

public class DiagnosticHttpServletResponseWrapper extends HttpServletResponseWrapper {

private CountingServletOutputStream countingStream;
private long startTime;

public DiagnosticHttpServletResponseWrapper(HttpServletResponse response) {
super(response);
countingStream = null;
startTime = System.currentTimeMillis();
}

public int getCount() {
return (countingStream != null ? countingStream.getCount(): 0);
}

public long getResponseTime() {
return System.currentTimeMillis() - startTime;
}

@Override
public PrintWriter getWriter() throws IOException {
return new PrintWriter(getOutputStream());
}

@Override
public ServletOutputStream getOutputStream() throws IOException {
if (countingStream == null) {
countingStream = new CountingServletOutputStream(super.getOutputStream());
}
return countingStream;
}
}
16 changes: 16 additions & 0 deletions webui/src/main/java/gov/nih/nlm/lode/servlet/ServletUtils.java
Original file line number Diff line number Diff line change
@@ -0,0 +1,16 @@
package gov.nih.nlm.lode.servlet;

import javax.servlet.http.HttpServletRequest;

public class ServletUtils {

public static String getClientAddress(HttpServletRequest request) {
String v;
if ((v = request.getHeader("X-Forwarded-For")) != null) {
// parse and return
return v.split(",")[0].trim();
}
return request.getRemoteAddr();
}

}
80 changes: 80 additions & 0 deletions webui/src/main/java/gov/nih/nlm/lode/servlet/SparqlController.java
Original file line number Diff line number Diff line change
@@ -0,0 +1,80 @@
package gov.nih.nlm.lode.servlet;

import java.io.IOException;
import java.time.Instant;
import java.time.ZoneId;
import java.time.ZonedDateTime;
import java.time.format.DateTimeFormatter;

import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
import javax.servlet.http.HttpSession;

import org.apache.log4j.MDC;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.stereotype.Controller;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RequestParam;
import org.springframework.web.bind.annotation.ResponseBody;

import com.hp.hpl.jena.query.QueryParseException;

import uk.ac.ebi.fgpt.lode.exception.LodeException;
import uk.ac.ebi.fgpt.lode.servlet.SparqlServlet;


@Controller
@RequestMapping("/query")
public class SparqlController extends SparqlServlet {

private Logger apilog = LoggerFactory.getLogger("gov.nih.nlm.lode.api");

@Override
@RequestMapping
public @ResponseBody
void query(
@RequestParam(value = "query", required = false) String query,
@RequestParam(value = "format", required = false) String format,
@RequestParam(value = "offset", required = false) Integer offset,
@RequestParam(value = "limit", required = false) Integer limit,
@RequestParam(value = "inference", required = false) boolean inference,
HttpServletRequest request,
HttpServletResponse response) throws QueryParseException, LodeException, IOException {
String v;
if ((v = request.getHeader("User-Agent")) != null)
MDC.put("ua", v);
if ((v = ServletUtils.getClientAddress(request)) != null)
MDC.put("cliaddr", v);
if ((v = request.getRequestedSessionId()) != null)
MDC.put("requestedsession", v);
v = request.getHeader("Referer");
MDC.put("webui", v != null && v.contains("/mesh/query"));
if (query != null)
MDC.put("query", query);
if (format != null)
MDC.put("format", format);
if (limit != null)
MDC.put("limit", limit);
if (offset != null)
MDC.put("offset", offset);
MDC.put("inference", inference);
HttpSession session = request.getSession();
if (session != null) {
// Convert session datetime into a zoned date time
ZonedDateTime zdt = ZonedDateTime.ofInstant(
Instant.ofEpochMilli(session.getCreationTime()),
ZoneId.systemDefault()
);
// print that as ISO8601 formatted timestamp
MDC.put("sessiontime", zdt.format(DateTimeFormatter.ISO_OFFSET_DATE_TIME));
MDC.put("sessionid", session.getId());
}
DiagnosticHttpServletResponseWrapper wrappedResponse = new DiagnosticHttpServletResponseWrapper(response);
super.query(query, format, offset, limit, inference, request, wrappedResponse);
MDC.put("responsesize", wrappedResponse.getCount());
MDC.put("responsetime", wrappedResponse.getResponseTime());
apilog.info("sparql query");
MDC.clear();
}
}
13 changes: 6 additions & 7 deletions webui/src/main/resources/log4j.xml
Original file line number Diff line number Diff line change
Expand Up @@ -3,17 +3,16 @@
<log4j:configuration xmlns:log4j="http://jakarta.apache.org/log4j/">

<appender name="console" class="org.apache.log4j.ConsoleAppender">
<layout class="org.apache.log4j.PatternLayout">
<layout name="console-layout" class="org.apache.log4j.PatternLayout">
<param name="ConversionPattern" value="%d %-5p [%t] %C{2} (%F:%L) - %m%n"/>
</layout>
</appender>

<appender name="file" class="org.apache.log4j.FileAppender">
<param name="file" value="${lodestar.log.prefix}lodestar.log" />
<param name="append" value="false" />
<layout class="org.apache.log4j.PatternLayout">
<param name="ConversionPattern" value="%d %-5p [%t] %C{2} (%F:%L) - %m%n"/>
</layout>
<appender name="file" class="org.apache.log4j.RollingFileAppender">
<param name="file" value="${lodestar.log.prefix}lodestar.jsonl" />
<param name="MaxFileSize" value="32MB"/>
<param name="MaxBackupIndex" value="5"/>
<layout name="file-layout" class="net.logstash.log4j.JSONEventLayoutV1" />
</appender>

<logger name="org.springframework" additivity="false">
Expand Down
14 changes: 9 additions & 5 deletions webui/src/main/webapp/WEB-INF/ebi-lode-service.xml
Original file line number Diff line number Diff line change
Expand Up @@ -12,13 +12,15 @@

<bean name="explorerConfig" class="uk.ac.ebi.fgpt.lode.impl.DefaultExplorerViewConfigImpl"/>

<!-- The Explorer Servlet is auto-wired by the lode-servlet.xml, this code is not really used -->
<bean id="serviceServlet" class="uk.ac.ebi.fgpt.lode.servlet.ExplorerServlet">
<!-- The Explorer Servlet is auto-wired by the lode-servlet.xml so that this bean
wiring is provided for completeness.
<bean id="serviceServlet" class="uk.ak.ebi.fgpt.servlet.ExplorerServlet">
<property name="sparqlService" ref="jenaSparqlService"/>
<property name="configuration" ref="explorerConfig"/>
<property name="service" ref="explorerServiceImpl"/>
<property name="baseUri" value="${lode.explorer.service.baseuri}"/>
</bean>
-->

<!--Uncomment these two beans for virtuoso connection pooling-->
<bean id="jenaVirtuosoConnectionPoolService" class="uk.ac.ebi.fgpt.lode.impl.JenaVirtuosoConnectionPoolService">
Expand All @@ -37,12 +39,11 @@
<!--<property name="maxPoolSize" value="10"/>-->
<!--</bean>-->

<!--For direct virtuoso JDBC connection, use this service-->
<!--For direct virtuoso JDBC connection, use this service-->
<!--<bean id="jenaVirtuosoExecutorService" class="uk.ac.ebi.fgpt.lode.impl.JenaVirtuosoExecutorService">-->
<!--<property name="endpointURL" value="jdbc:virtuoso://${lode.sparqlendpoint.url}:${lode.sparqlendpoint.port}" />-->
<!--</bean>-->


<!--
A basic HTTP service bean for connecting to SPARQL endpoint
-->
Expand All @@ -60,9 +61,12 @@
<property name="queryExecutionService" ref="jenaVirtuosoConnectionPoolService"/>
</bean>

<bean id="sparqlServlet" class="uk.ac.ebi.fgpt.lode.servlet.SparqlServlet">
<!-- This bean is wired by lode-servlet.xml, at the dispacher context level, but
is included here for completeness.
<bean id="sparqlServlet" class="gov.nih.nlm.lode.servlet.SparqlController">
<property name="sparqlService" ref="jenaSparqlService"/>
</bean>
-->

<bean id="queryReader" class="uk.ac.ebi.fgpt.lode.utils.SparqlQueryReader" init-method="init">
<property name="sparqlQueryResource" value="classpath:sparql-queries.txt" />
Expand Down
9 changes: 7 additions & 2 deletions webui/src/main/webapp/WEB-INF/lode-servlet.xml
Original file line number Diff line number Diff line change
Expand Up @@ -17,8 +17,13 @@
<!-- autowire this context only (just controllers, not services) based on annotations -->
<context:annotation-config />

<!-- Add also gov.nih.nlm.lode.servlet -->
<context:component-scan base-package="uk.ac.ebi.fgpt.lode.servlet, gov.nih.nlm.lode.servlet" />
<!-- Allow our sparqlServlet to override lodestar's base servlet -->
<context:component-scan base-package="uk.ac.ebi.fgpt.lode.servlet">
<context:exclude-filter type="regex" expression="uk\.ac\.ebi\.fgpt\.lode\.servlet\.SparqlServlet"/>
</context:component-scan>

<!-- Scan our own packages -->
<context:component-scan base-package="gov.nih.nlm.lode.servlet"/>

<!-- drive MVC setup based on @Controller, @RequestMapping annotations etc -->
<mvc:annotation-driven />
Expand Down
5 changes: 5 additions & 0 deletions webui/src/main/webapp/css/style.css
Original file line number Diff line number Diff line change
Expand Up @@ -159,3 +159,8 @@ ul#queries_list {
#example_queries h3 {
margin-top: 0;
}

.version {
float: right;
padding-right: 15px;
}
2 changes: 1 addition & 1 deletion webui/src/main/webapp/error.jsp
Original file line number Diff line number Diff line change
Expand Up @@ -58,7 +58,7 @@
</div>
<div class="footer">
<%@ include file="internal/footer.html" %>
<%@ include file="internal/footer.jspf" %>
</div>
</div>
</div>
Expand Down
2 changes: 1 addition & 1 deletion webui/src/main/webapp/explore.jsp
Original file line number Diff line number Diff line change
Expand Up @@ -33,7 +33,7 @@
</div>

<div class="footer">
<%@ include file="internal/footer.html" %>
<%@ include file="internal/footer.jspf" %>
</div>
</div>
</div>
Expand Down
2 changes: 1 addition & 1 deletion webui/src/main/webapp/index.jsp
Original file line number Diff line number Diff line change
Expand Up @@ -104,7 +104,7 @@
</div>
</div>
<div class="footer">
<%@ include file="internal/footer.html" %>
<%@ include file="internal/footer.jspf" %>
</div>
</div>
</div>
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -19,5 +19,10 @@
<img src="https://www.nlm.nih.gov/images/usagov_logo.gif" alt="USA.gov logo"/>
</a>
</div>
<div class="col-md-5 small">
<span class="version">
Version <%@ include file="/internal/about.html" %>
</span>
</div>
</div>
</footer>
</footer>
2 changes: 1 addition & 1 deletion webui/src/main/webapp/query.jsp
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,7 @@
</div>

<div class="footer">
<%@ include file="/internal/footer.html" %>
<%@ include file="/internal/footer.jspf" %>
</div>
</div>
</div>
Expand Down
Loading

0 comments on commit 94a793e

Please sign in to comment.