Aim:

collecting the log file from any one of the web server into Hdfs within the help of Apache Flume. The Collected log data is unstructured data. so we are making the data in structure Format by using Apache Hive. Once the Apache Hive is make the Structure data we run the Hive Udf(User Defined Function). it will count the number of user based on the country wise. Again we are display the result in jsp page using Apache JfreeChart.

1. code for collecting the log file from web server into Hdfs:

Step1: Flume configuration for collecting log file( vi /etc/flume-ng/conf/flume.conf)

# Name the components on this agent

agent1.sources = netcat-source1

agent1.sinks =log-sink1

agent1.channels = ch1

# Describe/configure source1

agent1.sources.netcat-source1.channels = ch1

agent1.sources.netcat-source1.type = netcat

agent1.sources.netcat-source1.bind =0.0.0.0

agent1.sources.netcat-source1.port = 45855

# Use a channel which buffers events in memory

agent1.channels.ch1.type = memory

agent1.channels.ch1.capacity = 200

# Bind the source and sink to the channel

agent1.sinks.log-sink1.channel = ch1

agent1.sinks.log-sink1.type = hdfs

agent1.sinks.log-sink1.hdfs.path=/user/log

agent1.sinks.log-sink1.hdfs.filePrefix=access

agent1.sinks.log-sink1.hdfs.fileSuffix=.log

agent1.sinks.log-sink1.hdfs.rollInterval=60

agent1.sinks.log-sink1.hdfs.rollCount=0

agent1.sinks.log-sink1.rollSize=0

Step2: Run the Below Command in Hadoop server

$ flume-ng agent --conf ./conf/ -f conf/flume.conf -Dflume.root.logger=DEBUG,console -n agent1

Step3: Run the below command in web server

$ cd /var/log/apache2

$ tail -f *.log | nc 192.237.180.29 45855

Sample Data:

82.95.120.160 - - [14/May/2009:00:10:34 -0700] "GET www.twibuzz.com/ HTTP/1.1" 200 112813 "-" "Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_4_11; en) AppleWebKit/528.16 (KHTML, like Gecko) Version/4.0 Safari/528.16"

82.95.120.160 - - [14/May/2009:00:10:35 -0700] "GET www.twibuzz.com/style.css HTTP/1.1" 200 639 "http://www.twibuzz.com/" "Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_4_11; en) AppleWebKit/528.16 (KHTML, like Gecko) Version/4.0 Safari/528.16"

82.95.120.160 - - [14/May/2009:00:10:36 -0700] "GET www.twibuzz.com/logo.png HTTP/1.1" 200 5410 "http://www.twibuzz.com/" "Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_4_11; en) AppleWebKit/528.16 (KHTML, like Gecko) Version/4.0 Safari/528.16"

82.95.120.160 - - [14/May/2009:00:10:43 -0700] "GET www.twibuzz.com/favicon.ico HTTP/1.1" 404 - "http://www.twibuzz.com/" "Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_4_11; en) AppleWebKit/528.16 (KHTML, like Gecko) Version/4.0 Safari/528.16"

66.249.70.102 - - [14/May/2009:00:35:33 -0700] "GET www.twibuzz.com/cgi/tweets.py?q=denver HTTP/1.1" 200 7322 "-" "Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)"

66.249.70.102 - - [14/May/2009:00:36:20 -0700] "GET www.twibuzz.com/cgi/tweets.py?q=nba HTTP/1.1" 200 5898 "-" "Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)"

2. Code for converting the un structure data into structure data

$ su hdfs

$ hive

hive > CREATE EXTERNAL TABLE access(

host STRING,

identity STRING,

user STRING,

time STRING,

request STRING,

status STRING,

size STRING,

referer STRING,

agent STRING)

ROW FORMAT SERDE 'org.apache.hadoop.hive.contrib.serde2.RegexSerDe'

WITH SERDEPROPERTIES (

"input.regex" = "([^ ]*) ([^ ]*) ([^ ]*) (-|\\[[^\\]]*\\]) ([^ \"]*|\"[^\"]*\") (-|[0-9]*) (-|[0-9]*)(?: ([^ \"]*|\"[^\"]*\") ([^ \"]*|\"[^\"]*\"))?",

"output.format.string" = "%1$s %2$s %3$s %4$s %5$s %6$s %7$s %8$s %9$s"

)

STORED AS TEXTFILE

LOCATION '/user/log';

3. code for converting the IP address into country

· Hive udf code

package com.jointhegrid.udf.geoip;

import java.io.File;

import java.io.IOException;

import java.util.Arrays;

import java.util.Set;

import java.util.concurrent.CopyOnWriteArraySet;

import org.apache.hadoop.hive.ql.exec.UDFArgumentException;

import org.apache.hadoop.hive.ql.exec.UDFArgumentLengthException;

import org.apache.hadoop.hive.ql.exec.UDFArgumentTypeException;

import org.apache.hadoop.hive.ql.metadata.HiveException;

import org.apache.hadoop.hive.ql.udf.generic.GenericUDF;

import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;

import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector;

import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector.PrimitiveCategory;

import org.apache.hadoop.hive.serde2.objectinspector.primitive.LongObjectInspector;

import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory;

import org.apache.hadoop.hive.serde2.objectinspector.primitive.StringObjectInspector;

import com.maxmind.geoip.Country;

import com.maxmind.geoip.LookupService;

public class GenericUDFGeoIP extends GenericUDF {

private String ipString = null;

private Long ipLong = null;

private String database;

private String property;

private LookupService ls;

private static final String COUNTRY_NAME = "COUNTRY_NAME";

private static final Set<String> COUNTRY_PROPERTIES =

new CopyOnWriteArraySet<String>(Arrays.asList(

new String[] {COUNTRY_NAME}));

PrimitiveObjectInspector [] argumentOIs;

public ObjectInspector initialize(ObjectInspector[] arguments)

throws UDFArgumentException {

argumentOIs = new PrimitiveObjectInspector [arguments.length];

if ( arguments.length != 3) {

throw new UDFArgumentLengthException(

"The function GenericUDFGeoIP( 'input','resultfield','datafile' ) "

+ " accepts 3 arguments.");

}

if (!(arguments[0] instanceof StringObjectInspector) && !(arguments[0] instanceof LongObjectInspector)) {

throw new UDFArgumentTypeException(0,

"The first 3 parameters of GenericUDFGeoIP('input', 'resultfield', 'datafile')"

+ " should be string.");

}

argumentOIs[0] = (PrimitiveObjectInspector) arguments[0];

for (int i = 1; i < arguments.length; i++) {

if (!(arguments[i] instanceof StringObjectInspector )) {

throw new UDFArgumentTypeException(i,

"The first 3 parameters of GenericUDFGeoIP('input', 'resultfield', 'datafile')"

+ " should be string.");

}

argumentOIs[i] = (StringObjectInspector) arguments[i];

}

return PrimitiveObjectInspectorFactory.getPrimitiveJavaObjectInspector(

PrimitiveCategory.STRING);

}

public Object evaluate(DeferredObject[] arguments) throws HiveException {

if (argumentOIs[0] instanceof LongObjectInspector) {

this.ipLong = ((LongObjectInspector)argumentOIs[0]).get(arguments[0].get());

} else {

this.ipString = ((StringObjectInspector)argumentOIs[0]).getPrimitiveJavaObject(arguments[0].get());

}

this.property = ((StringObjectInspector)argumentOIs[1]).getPrimitiveJavaObject(arguments[1].get());

if (this.property != null) {

this.property = this.property.toUpperCase();

}

if (ls ==null){

if (argumentOIs.length == 3){

this.database = ((StringObjectInspector)argumentOIs[1]).getPrimitiveJavaObject(arguments[2].get());

File f = new File(database);

if (!f.exists()){

throw new HiveException(database+" does not exist");

}

try {

ls = new LookupService ( f , LookupService.GEOIP_MEMORY_CACHE );

} catch (IOException ex){

throw new HiveException (ex);

}

if (COUNTRY_PROPERTIES.contains(this.property)) {

Country country = ipString != null ? ls.getCountry(ipString) : ls.getCountry(ipLong);

if (country == null) {

return null;

} else if (this.property.equals(COUNTRY_NAME)) {

return country.getName();

}

assert(false);

}

return null;

}

@Override

public String getDisplayString(String[] children) {

assert(children.length == 3);

return "IptoCountry ( "+children[0]+", "+children[1]+", "+children[2]+")";

}

· Make this file as a jar. and transfer file into hdfs. with the file dependencies

$ su hdfs

hive

> add file /opt/GeoIP.dat;

> ADD JAR /opt/dspace-geoip-1.2.3.jar;

> ADD JAR /opt/IptoCountry.jar;

> CREATE TEMPORARY FUNCTION IptoCountry AS 'com.jointhegrid.udf.geoip.GenericUDFGeoIP';

> set hive.stats.autogather = false;

> CREATE TABLE apache_log_result(

country STRING,

visitors INT);

> INSERT OVERWRITE TABLE apache_log_result select IptoCountry(host, 'COUNTRY_NAME','./GeoIP.dat')as country ,count(host) as visit from access group by IptoCountry(host, 'COUNTRY_NAME','./GeoIP.dat');

4. code for display the Result in jfree chart

Servelet code:

* To change this template, choose Tools | Templates

* and open the template in the editor.

package com;

import java.awt.BasicStroke;

import java.awt.Color;

import java.io.IOException;

import java.io.OutputStream;

import java.sql.Connection;

import java.sql.DriverManager;

import java.sql.SQLException;

import javax.servlet.ServletException;

import javax.servlet.http.HttpServlet;

import javax.servlet.http.HttpServletRequest;

import javax.servlet.http.HttpServletResponse;

import org.jfree.chart.ChartFactory;

import org.jfree.chart.ChartRenderingInfo;

import org.jfree.chart.ChartUtilities;

import org.jfree.chart.JFreeChart;

import org.jfree.chart.entity.StandardEntityCollection;

import org.jfree.chart.plot.PlotOrientation;

import org.jfree.data.jdbc.JDBCCategoryDataset;

public class BarChartTry extends HttpServlet {

private static final long serialVersionUID = 1L;

public BarChartTry() {

// TODO Auto-generated constructor stub

}

protected void doGet(HttpServletRequest request,

HttpServletResponse response) throws ServletException, IOException {

Connection connection = null;

try {

Class.forName("com.mysql.jdbc.Driver").newInstance();

try {

connection = DriverManager

.getConnection("jdbc:mysql://50.56.72.45:3306/vms?user=root&password=root&useUnicode=true&characterEncoding=utf-8");

} catch (SQLException e) {

e.printStackTrace();

}

} catch (InstantiationException e) {

e.printStackTrace();

} catch (IllegalAccessException e) {

e.printStackTrace();

} catch (ClassNotFoundException e) {

e.printStackTrace();

}

// JDBCPieDataset dataset = new JDBCPieDataset(connection);

JDBCCategoryDataset dataset = new JDBCCategoryDataset(connection);

try {

dataset.executeQuery("Select country,visitor From log_result");

JFreeChart chart = ChartFactory.createBarChart(

"Apache Log viewer Deatils in country wise", "country",

"count", dataset, PlotOrientation.HORIZONTAL, true, true,

true);

chart.setBorderPaint(Color.black);

chart.setBorderStroke(new BasicStroke(10.0f));

chart.setBorderVisible(true);

if (chart != null) {

int width = 1500;

int height = 1000;

final ChartRenderingInfo info = new ChartRenderingInfo(

new StandardEntityCollection());

response.setContentType("image/png");

OutputStream out = response.getOutputStream();

ChartUtilities.writeChartAsPNG(out, chart, width, height, info);

}

} catch (SQLException e) {

e.printStackTrace();

}

protected void doPost(HttpServletRequest request,

HttpServletResponse response) throws ServletException, IOException {

// TODO Auto-generated method stub

}

jsp code:

<%@ page language="java" contentType="text/html; charset=ISO-8859-1"

pageEncoding="ISO-8859-1"%>

<!DOCTYPE html PUBLIC "-//W3C//DTD HTML 4.01 Transitional//EN" "http://www.w3.org/TR/html4/loose.dtd">

<html>

<head>

<title>Apache Log File Information</title>

</head>

<body>

</body>

</html>

Output of the Project is:

Hadoopfreelearning

Thursday, 5 September 2013

Apache Log File Analysis in hadoop

Aim:

No comments:

Post a Comment

About Me