Tuesday, 24 September 2013

Get the noun from the text using The Stanford Natural Language Processing

Hi Guys,
          Today my task is i need to extract the noun from the doc file. so i did some research on the topics. finally i found  solution for the task. so i am sharing the code with you.

package com.srini.experiment;

import java.io.File;
import java.io.IOException;
import java.util.StringTokenizer;
import java.util.TreeSet;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

import org.apache.tika.Tika;
import org.apache.tika.exception.TikaException;

import edu.stanford.nlp.tagger.maxent.MaxentTagger;

public class GetNoun {
public static void main(String[] args) throws IOException, TikaException {
MaxentTagger tagger = new MaxentTagger(
"D:/hadoopbooks/apachetika/wsj-0-18-bidirectional-nodistsim.tagger");
String con = new Tika().parseToString(new File(
"D:/hadoopbooks/apachetika/Srinivasan.doc"));
String string = con.replaceAll("[^\\p{L}\\p{Nd}]", " ");
String string2 = string.replaceAll("\\s+", " ");
String lo = string2.toLowerCase();
TreeSet<String> set = new TreeSet<String>();
StringTokenizer tokenizer = new StringTokenizer(lo);
while (tokenizer.hasMoreElements()) {
String word = (String) tokenizer.nextElement();
String tagged = tagger.tagString(word);
Pattern p = Pattern.compile(".*NN");
Matcher m = p.matcher(tagged);
if(m.find()){
String string3=m.group(0);
String noun=string3.replace("_NN", "");
set.add(noun);
}
}
System.out.println(set);

}

}

                    

Friday, 6 September 2013

My Mysql Learning Work part-1

MySql Exmple:
 
 1.Copying the table in mysql
     Syntax:
      Create table <newtablename> select * from <oldtablename>
ex: create table srini select * from ach_state_master  

 2.Cpoying only selected table in mysql
     syntax:
          create table <newtablename> select <selected columns> from  <oldtablename>
ex:create table srini2 select ach_sm_state_id, ach_sm_cntry_id from ach_state_master

 3. Display the Mysql version
    $ select version();

 4. display the used databases name
    $ select database();

 5.specify whic row to select(where condtion is used for this statement)
    $  select * from vms_candidate where can_firstname='santhosh'
$  select * from vms_candidate where can_firstname like 's%'

 6. display the data as your wish
    $ select concat(monthname(submitted_date),' ',dayofmonth(submitted_date),',',year(submitted_date)) from vms_candidate;
$ select date_format(submitted_date,'%M,%e,%Y') from vms_candidate;

 7. WHERE Clauses and Column Aliases
   $ select date_format(submitted_date,'%Y') from vms_candidate where date_format(submitted_date,'%Y')=2013;
   $ select * from vms_candidate where  date_format(submitted_date,'%Y') like '2%'
   $ select  date_format(submitted_date,'%M,%e,%Y') from vms_candidate where  date_format(submitted_date,'%Y') like '2%'

8. write the condition without using where condtion(if the condition is satisfied it will give boolean value otherwise it will give 0 valu)
  $  select date_format(submitted_date,'%M,%e,%Y') < 2014 from vms_candidate;
  $  select date_format(submitted_date,'%M,%e,%Y') > 2014 from vms_candidate;
  $ select DISTINCT user_name='asdfs' from vms_admin_users;
  $ select DISTINCT user_name='joseph' and password='ait123' from vms_admin_users;
  $ select count(DISTINCT user_name='joseph')  from vms_admin_users;

9. working with null
  $ select * from vms_candidate where submitted_date =null;
  $ select * rom vms_candidate where submitted_date !=null;(both of them dosn't dipaly any value)
 correct:
  $  select * from vms_candidate where submitted_date is null;
  $  select * from vms_candidate where submitted_date is not null;
  $  select if(id is null,'unknown',id) from vms_candidate ;
  $ select ifnull(id,'unknown') from vms_candidate;

10.Sorting a Result Set (order by command is used)
 $ select can_firstname from vms_candidate order by can_firstname;
 $ select can_firstname,can_lastname from vms_candidate order by can_firstname,can_lastname;
 
11.Using Views to Simplify Table Access();(we can't able to edit the view tbale);
 $  CREATE VIEW mail_view AS  select can_firstname from vms_candidate order by can_firstname;

12.Selecting Data from More Than One Table
 $ select * from  vms_candidate where user_id= (select admintypeid from admintype_menuoptions where admintypeid=3)

13.Selecting Rows from the Beginning or End of a Result Set(limit command is used)
 $  select * from  vms_candidate limit 5; -display the first five value)
 $  select * from  vms_candidate limit 4,5; -display the five value starts from 4th value)

14.create new table based on oldtable:
 syntax:
    create table <new table> like <oldtable>
  $ create table vms_candidate1 like vms_candidate;

15.insert the value from old table into new table
  $ insert into  vms_candidate1 select * from vms_candidate where can_firstname='santhosh'
 
16.create the temporary table - when you close the server connection the temp table delete it self)
  $ create temporary table vms_candidate select can_firstname from vms_candidate order by can_firstname
  $ select * from name;

17.Checking or Changing a Table’s Storage Engine
  $  SHOW TABLE STATUS LIKE 'vms_candidate' -display the table status like engine name)
  $  SHOW CREATE TABLE vms_candidate; -(get the tabe inormation)
  $ ALTER TABLE vms_candidate ENGINE = InnoDB; (chnage the engine name)
 
18.Find the character length:
 $ select char_length(can_firstname) from vms_candidate;
 $ select * from vms_candidate CHAR_LENGTH( can_firstname ) =8;

19.Writing String Literals
 $ select i'm srinivasan'; -it show erorr because our quotation is wrong
 $ select "i'm srinivasan";

20.Checking a String’s Character Set or Collation
 $ select user(),charset(user()),collation(user());

21.Display the character set:
 $ show character set;
 $ SHOW COLLATION LIKE 'latin1%';

22.set the character set
 $ set names 'latin1'
 $  SELECT CHARSET('abc'), COLLATION('abc');

23.convert the string in uppercase or lowercase
 $ select upper(can_firstname) from vms_candidate;
 $ select lower(can_firstname) from vms_candidate;
 $ select concat(upper(can_firstname),lower(can_firstname)) from vms_candidate;

24.Display the initial letter in cap
 $ select inital_cap(can_firstname) from vms_candidate;

25.Pattern Matching with SQL Patterns
 End letter matching

   $ select can_lastname from vms_candidate where can_lastname like '%r'

 Firsttwo line matching
   $ select can_lastname from vms_candidate where can_lastname like 'ku%'

 middle character matching
    $ select can_lastname from vms_candidate where can_lastname like '%ma%'
third postion character matching
    $ select can_lastname from vms_candidate where can_lastname like '__m%'

26. display the last two character :
    $ select right(can_lastname,2) from vms_candidate;
27.display the first three character:
    $ select left(can_lastname,3) from vms_candidate;
$ select concat(left(can_lastname,3),right(can_lastname,1)) from vms_candidate;
28.Pattern Matching with Regular Expressions
  Strings that begin with a particular substring
    $ select can_lastname from vms_candidate where can_lastname regexp '^ku'
 
  Strings that end with a particular substring:
    $ select can_lastname from vms_candidate where can_lastname regexp 'ar$'  
 
  Strings that contain a particular substring at any position:
    $ select can_lastname from vms_candidate where can_lastname regexp 'ar'
 
  Strings that contain a particular substring at a specific position:
    $ select can_lastname from vms_candidate where can_lastname regexp '^...ar'

29.Breaking Apart or Combining Strings( use left(),right(),mid(),concat(),substring_index())
   display the last two character
    $ select right(can_lastname,2) from vms_candidate;  
   display the midel character:
    $ select mid(can_lastname,2,4) from vms_candidate;
   display the first three character
    $ select left(can_lastname,3) from vms_candidate;
   display the two column information in one column
    $ select concat(can_lastname,' ',can_firstname) from vms_candidate;
   The substring_index(columname,matchedchar,'positive or negative value')
    positive value- display the right side character after matching the character
    negative value - display the let side character after matching the character
   $ select substring_index(can_lastname,'m',1) from vms_candidate;
   The substring-substring(columnanem,lenght)
   $ select substring(can_lastname,2) from vms_candidate;
   $ select substring(can_lastname,2,2) from vms_candidate;
 
 30.Searching for Substrings
  locate()-function is used for searching the substring
  $ select locate('ma',can_lastname) from vms_candidate;
  $ select locate ('ma',conact(can_firstname,' ',lastname)') from vms_candidate;

 31.Using FULLTEXT Searches
  $ ALTER TABLE vms_candidate ADD FULLTEXT (can_firstname);
  $ SELECT count(*) from vms_candidate WHERE MATCH(can_firstname) AGAINST('santhosh');



 <!------------------Working with Dates and Times--------->
 1.Changing MySQL’s Date Format using (date_format(columname,'abbervation'),timeformat(columname,'abberivation'))
   $ select date_format(submitted_date,'%M,%e,%y') from vms_candidate;
   $ select time_format(submitted_date,'%r') as '12-hourstime',time_format(submitted_date,'%T') as '24-hourstime' from vms_candidate;
 
 2.Determining the Current Date or Time
  display the current date(),currenttime and display the date and time
  $ select curdate(),curtime(),now()
  $ select UTC_DATE();
  $ select UTC_TIME();
  $ select utc_timestamp;

3.Extracting Parts of Dates or Times
   Diplay the date
  $ select date(submitted_date) from vms_candidate;
  Display the time
  $ select time(submitted_date) from vms_candidate;
  Display the year
  $ select year(submitted_date) from vms_candidate; --like 2013
  Display the month name
  $ select monthname(submitted_date) from vms_candidate; --like(feb)
  Display the month in integer
  $ select month(submitted_date) from vms_candidate; --like(1-12)
  Display the day of month
   $ select dayofmonth(submitted_date) as date from vms_candidate order by date  ;  --like( 1..31)
  Display the day of week
  $ select dayofweek(submitted_date) from vms_candidate;   -(1 to 7)
  Display the day name
   $ select dayname(submitted_date) from vms_candidate;  -like monday
  Display the day of year
 $ select dayofyear(submitted_date) from vms_candidate;  like 218days
 Display the weekday
  $ select weekday(submitted_date) from vms_candidate; --(0--6)
 Display the hours
 $ select hour(submitted_date) from vms_candidate;
 $ select second()
 $ select minute()

4. Converting Between Temporal Data Types and Basic Units
  change the time into seconds
  $ select time_to_sec(curtime())
  chanage the sec to time
  $ select sec_to_time()

5.Converting between dates and days
   convert date to days
   $ select to_days(submitted_date) from vms_candidate;
  convert days to date
  $ select from_days(to_days(submitted_date)) from vms_candidate;

6. covert the date into unix time stamp
  $ select unix_timestamp(submitted_date) from vms_candidate;
7.convert the unix timestamp into date
 $ select from_unixtime(unix_timestamp(submitted_date)) from vms_candidate;

8.Calculating the Interval Between Two Dates or Times
  $ select datediff(curdate(),submitted_date) from vms_candidate;
  $ select timediff(time(submitted_date),curtime()) from vms_candidate;

9.Adding Date or Time Values
   $ select  addtime(time(submitted_date),curtime()) from vms_candidate;
   $ select date_add(curdate(),INTERVAL 3 DAY) from vms_candidate;
   $ select date_add(submitted_date,INTERVAL 3 DAY) from vms_candidate; display the date in after three days
   $ select date_add(submitted_date,INTERVAL -3 DAY) from vms_candidate; display the date in befter three days
 
10.Calculating Ages
  $ select timestampdiff(year,'1989-09-17',curdate()) from vms_candidate;  
  $ select timestampdiff(year,submitted_date,curdate()) from vms_candidate;

11.calculating Age in the months
  $ select timestampdiff(month,submitted_date,curdate()) from vms_candidate;
12.calculaing age in day
  $ select timestampdiff(day,submitted_date,curdate()) from vms_candidate;
  $ select timestampdiff(day,'1989-09-17',curdate()) from vms_candidate;  
 
13. Finding the First Day, Last Day, or Length of a Month
$ select submitted_date,date_sub(submitted_date,interval dayofmonth(submitted_date)-1 day) from vms_candidate;
$ select last_day(submitted_date) from vms_candidate;

14.Finding sunday in the particular week
  $  select date_add(submitted_date,INTERVAL 1-dayofweek(submitted_date) DAY) from vms_candidate;

15.Finding saturday in th particular week
  $ select dayname(date_add(submitted_date,INTERVAL 7-dayofweek(submitted_date) DAY)) from vms_candidate;
  

Thursday, 5 September 2013

Apache Log File Analysis in hadoop

Aim:

       collecting the log file from any one of the web server into Hdfs within the help of  Apache Flume. The Collected log data is unstructured data. so we are making the data in structure Format  by using Apache Hive. Once the Apache Hive is make the Structure data we run the Hive Udf(User Defined Function). it will count the number of user based on the country wise. Again we  are display the result in jsp page using Apache JfreeChart.
1. code for collecting the log file from web server into Hdfs:
            Step1: Flume configuration for collecting log file( vi /etc/flume-ng/conf/flume.conf)
                # Name the components on this agent
                agent1.sources = netcat-source1
                agent1.sinks =log-sink1
                agent1.channels = ch1

                # Describe/configure source1
                agent1.sources.netcat-source1.channels = ch1
                agent1.sources.netcat-source1.type = netcat
                agent1.sources.netcat-source1.bind =0.0.0.0
                agent1.sources.netcat-source1.port = 45855

                # Use a channel which buffers events in memory
                agent1.channels.ch1.type = memory
                agent1.channels.ch1.capacity = 200

                # Bind the source and sink to the channel
                agent1.sinks.log-sink1.channel = ch1
                agent1.sinks.log-sink1.type = hdfs
                agent1.sinks.log-sink1.hdfs.path=/user/log
                agent1.sinks.log-sink1.hdfs.filePrefix=access
                agent1.sinks.log-sink1.hdfs.fileSuffix=.log
                agent1.sinks.log-sink1.hdfs.rollInterval=60
                agent1.sinks.log-sink1.hdfs.rollCount=0
                agent1.sinks.log-sink1.rollSize=0
Step2:  Run the Below Command in Hadoop server
   $  flume-ng agent --conf ./conf/ -f conf/flume.conf -Dflume.root.logger=DEBUG,console -n agent1
Step3: Run the below command in web server
  $  cd /var/log/apache2
 $  tail -f  *.log | nc 192.237.180.29 45855


Sample Data:
82.95.120.160 - - [14/May/2009:00:10:34 -0700] "GET www.twibuzz.com/ HTTP/1.1" 200 112813 "-" "Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_4_11; en) AppleWebKit/528.16 (KHTML, like Gecko) Version/4.0 Safari/528.16"
82.95.120.160 - - [14/May/2009:00:10:35 -0700] "GET www.twibuzz.com/style.css HTTP/1.1" 200 639 "http://www.twibuzz.com/" "Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_4_11; en) AppleWebKit/528.16 (KHTML, like Gecko) Version/4.0 Safari/528.16"
82.95.120.160 - - [14/May/2009:00:10:36 -0700] "GET www.twibuzz.com/logo.png HTTP/1.1" 200 5410 "http://www.twibuzz.com/" "Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_4_11; en) AppleWebKit/528.16 (KHTML, like Gecko) Version/4.0 Safari/528.16"
82.95.120.160 - - [14/May/2009:00:10:43 -0700] "GET www.twibuzz.com/favicon.ico HTTP/1.1" 404 - "http://www.twibuzz.com/" "Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_4_11; en) AppleWebKit/528.16 (KHTML, like Gecko) Version/4.0 Safari/528.16"
66.249.70.102 - - [14/May/2009:00:35:33 -0700] "GET www.twibuzz.com/cgi/tweets.py?q=denver HTTP/1.1" 200 7322 "-" "Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)"
66.249.70.102 - - [14/May/2009:00:36:20 -0700] "GET www.twibuzz.com/cgi/tweets.py?q=nba HTTP/1.1" 200 5898 "-" "Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)"
2. Code for converting the un structure data into structure data  
$ su hdfs
$ hive
hive  >  CREATE EXTERNAL TABLE access(
host STRING,
identity STRING,
user STRING,
time STRING,
request STRING,
status STRING,
size STRING,
referer STRING,
agent STRING)
ROW FORMAT SERDE 'org.apache.hadoop.hive.contrib.serde2.RegexSerDe'
WITH SERDEPROPERTIES (
"input.regex" = "([^ ]*) ([^ ]*) ([^ ]*) (-|\\[[^\\]]*\\]) ([^ \"]*|\"[^\"]*\") (-|[0-9]*) (-|[0-9]*)(?: ([^ \"]*|\"[^\"]*\") ([^ \"]*|\"[^\"]*\"))?",
"output.format.string" = "%1$s %2$s %3$s %4$s %5$s %6$s %7$s %8$s %9$s"
)
STORED AS TEXTFILE
LOCATION '/user/log';

3. code for converting the IP address into country
·         Hive udf code
package com.jointhegrid.udf.geoip;

import java.io.File;
import java.io.IOException;
import java.util.Arrays;
import java.util.Set;
import java.util.concurrent.CopyOnWriteArraySet;
import org.apache.hadoop.hive.ql.exec.UDFArgumentException;
import org.apache.hadoop.hive.ql.exec.UDFArgumentLengthException;
import org.apache.hadoop.hive.ql.exec.UDFArgumentTypeException;
import org.apache.hadoop.hive.ql.metadata.HiveException;
import org.apache.hadoop.hive.ql.udf.generic.GenericUDF;
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector;
import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector.PrimitiveCategory;
import org.apache.hadoop.hive.serde2.objectinspector.primitive.LongObjectInspector;
import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory;
import org.apache.hadoop.hive.serde2.objectinspector.primitive.StringObjectInspector;
import com.maxmind.geoip.Country;
import com.maxmind.geoip.LookupService;

public class GenericUDFGeoIP extends GenericUDF {
                  private String ipString = null;
                  private Long ipLong = null;
                  private String database;
                  private String property;
                  private LookupService ls;
                  private static final String COUNTRY_NAME = "COUNTRY_NAME";
                  private static final Set<String> COUNTRY_PROPERTIES =
                                  new CopyOnWriteArraySet<String>(Arrays.asList(
                                  new String[] {COUNTRY_NAME}));
                 
                  PrimitiveObjectInspector [] argumentOIs;

                public ObjectInspector initialize(ObjectInspector[] arguments)
                                                throws UDFArgumentException {
                                argumentOIs = new PrimitiveObjectInspector [arguments.length];

                    if ( arguments.length != 3) {
                      throw new UDFArgumentLengthException(
                              "The function GenericUDFGeoIP( 'input','resultfield','datafile' ) "
                              + " accepts 3 arguments.");
                    }
                    if (!(arguments[0] instanceof StringObjectInspector) && !(arguments[0] instanceof LongObjectInspector)) {
                        throw new UDFArgumentTypeException(0,
                                 "The first 3 parameters of GenericUDFGeoIP('input', 'resultfield', 'datafile')"
                                 + " should be string.");
                      }
                      argumentOIs[0] = (PrimitiveObjectInspector) arguments[0];

                      for (int i = 1; i < arguments.length; i++) {
                        if (!(arguments[i] instanceof StringObjectInspector )) {
                          throw new UDFArgumentTypeException(i,
                                  "The first 3 parameters of GenericUDFGeoIP('input', 'resultfield', 'datafile')"
                                  + " should be string.");
                        }
                        argumentOIs[i] = (StringObjectInspector) arguments[i];
                      }
                      return PrimitiveObjectInspectorFactory.getPrimitiveJavaObjectInspector(
                              PrimitiveCategory.STRING);
                }


                public Object evaluate(DeferredObject[] arguments) throws HiveException {
                    if (argumentOIs[0] instanceof LongObjectInspector) {
                      this.ipLong = ((LongObjectInspector)argumentOIs[0]).get(arguments[0].get());
                    } else {
                      this.ipString = ((StringObjectInspector)argumentOIs[0]).getPrimitiveJavaObject(arguments[0].get());
                    }
                    this.property = ((StringObjectInspector)argumentOIs[1]).getPrimitiveJavaObject(arguments[1].get());

                    if (this.property != null) {
                      this.property = this.property.toUpperCase();
                    }

                    if (ls ==null){
                      if (argumentOIs.length == 3){
                        this.database = ((StringObjectInspector)argumentOIs[1]).getPrimitiveJavaObject(arguments[2].get());
                        File f = new File(database);
                        if (!f.exists()){
                          throw new HiveException(database+" does not exist");
                        }
                        try {
                          ls = new LookupService ( f , LookupService.GEOIP_MEMORY_CACHE );
                        } catch (IOException ex){
                          throw new HiveException (ex);
                        }
                      }
                    
                    }

                    if (COUNTRY_PROPERTIES.contains(this.property)) {
                      Country country = ipString != null ? ls.getCountry(ipString) : ls.getCountry(ipLong);
                      if (country == null) {
                        return null;
                      } else if (this.property.equals(COUNTRY_NAME)) {
                        return country.getName();
                      }
                      assert(false);
                    } 

                    return null;
                  }
                @Override
                public String getDisplayString(String[] children) {
                                assert(children.length == 3);
                    return "IptoCountry ( "+children[0]+", "+children[1]+", "+children[2]+")";
                }

}

·         Make this file as a jar. and transfer file into hdfs. with the file dependencies
       $ su hdfs
        hive
      >    add file /opt/GeoIP.dat;
      >    ADD JAR /opt/dspace-geoip-1.2.3.jar;
      >    ADD JAR /opt/IptoCountry.jar;
      >    CREATE TEMPORARY FUNCTION IptoCountry AS 'com.jointhegrid.udf.geoip.GenericUDFGeoIP';
>  set hive.stats.autogather = false;
>    CREATE  TABLE apache_log_result(
country STRING,
visitors INT);
>  INSERT OVERWRITE TABLE apache_log_result select IptoCountry(host, 'COUNTRY_NAME','./GeoIP.dat')as country ,count(host) as visit from access group by IptoCountry(host, 'COUNTRY_NAME','./GeoIP.dat');

4. code for  display the Result in jfree chart
Servelet code:
/*
 * To change this template, choose Tools | Templates
 * and open the template in the editor.
 */
package com;

import java.awt.BasicStroke;
import java.awt.Color;
import java.io.IOException;
import java.io.OutputStream;
import java.sql.Connection;
import java.sql.DriverManager;
import java.sql.SQLException;

import javax.servlet.ServletException;
import javax.servlet.http.HttpServlet;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;

import org.jfree.chart.ChartFactory;
import org.jfree.chart.ChartRenderingInfo;
import org.jfree.chart.ChartUtilities;
import org.jfree.chart.JFreeChart;
import org.jfree.chart.entity.StandardEntityCollection;
import org.jfree.chart.plot.PlotOrientation;
import org.jfree.data.jdbc.JDBCCategoryDataset;

public class BarChartTry extends HttpServlet {

                private static final long serialVersionUID = 1L;

                public BarChartTry() {
                                // TODO Auto-generated constructor stub
                }

                protected void doGet(HttpServletRequest request,
                                                HttpServletResponse response) throws ServletException, IOException {
                                Connection connection = null;
                                try {
                                                Class.forName("com.mysql.jdbc.Driver").newInstance();
                                                try {
                                                                connection = DriverManager
                                                                                                .getConnection("jdbc:mysql://50.56.72.45:3306/vms?user=root&password=root&useUnicode=true&characterEncoding=utf-8");
                                                } catch (SQLException e) {
                                                                e.printStackTrace();
                                                }
                                } catch (InstantiationException e) {
                                                e.printStackTrace();
                                } catch (IllegalAccessException e) {
                                                e.printStackTrace();
                                } catch (ClassNotFoundException e) {
                                                e.printStackTrace();
                                }
                                // JDBCPieDataset dataset = new JDBCPieDataset(connection);
                                JDBCCategoryDataset dataset = new JDBCCategoryDataset(connection);
                                try {
                                                dataset.executeQuery("Select country,visitor From log_result");
                                                JFreeChart chart = ChartFactory.createBarChart(
                                                                                "Apache Log viewer Deatils in country wise", "country",
                                                                                "count", dataset, PlotOrientation.HORIZONTAL, true, true,
                                                                                true);
                                                chart.setBorderPaint(Color.black);
                                                chart.setBorderStroke(new BasicStroke(10.0f));
                                                chart.setBorderVisible(true);
                                                if (chart != null) {
                                                                int width = 1500;
                                                                int height = 1000;
                                                                final ChartRenderingInfo info = new ChartRenderingInfo(
                                                                                                new StandardEntityCollection());
                                                                response.setContentType("image/png");
                                                                OutputStream out = response.getOutputStream();
                                                                ChartUtilities.writeChartAsPNG(out, chart, width, height, info);
                                                }
                                } catch (SQLException e) {
                                                e.printStackTrace();
                                }
                }

                protected void doPost(HttpServletRequest request,
                                                HttpServletResponse response) throws ServletException, IOException {
                                // TODO Auto-generated method stub
                }
}



jsp code:

<%@ page language="java" contentType="text/html; charset=ISO-8859-1"
    pageEncoding="ISO-8859-1"%>
<!DOCTYPE html PUBLIC "-//W3C//DTD HTML 4.01 Transitional//EN" "http://www.w3.org/TR/html4/loose.dtd">
<html>
<head>
<meta http-equiv="Content-Type" content="text/html; charset=ISO-8859-1">
<title>Apache Log File Information</title>
</head>
<body>
<!--  <img src="chart" width="600" height="400" border="0"/>-->
<img src="BarChartTry" width="1200" height="600" border="0"/>
</body>
</html>

 Output of the Project is:


Friday, 23 August 2013

How to Dump the mysql database into hbase using sqoop:

How to Dump the mysql database into hbase using sqoop:

                      Hi Guys yesterday my task is Dump the mysql value into hbase table. so i was dump the mysql table into hbase table. by using the below query is done my job

Step1: create and grant the privillage in mysql server
        CREATE USER 'root'@'xxx.xxx.xxx.xxx' IDENTIFIED BY 'root';
        GRANT ALL PRIVILEGES ON *.* TO 'root'@'xxx.xxx.xxx.xxx' WITH GRANT OPTION;
        flush privileges;
step2: Create the table and column family in hbase
       $ hbase shell
         hbase > create  'cities', 'world'
step3: add the mysql connector in the sqoop lib folder
        $ cd /usr/lib/sqoop/lib
        $ wget https://jybookshop.googlecode.com/files/mysql-connector-java-5.1.6-bin.jar
 
step3:run the sqoop command
      $ su hdfds
      $ sqoop import -Dsqoop.hbase.add.row.key=true --connect jdbc:mysql://xx.xx.xxx.xxx:3306/vms --username root --password root --table vms_country --hbase-table cities --column-family world

      When i try to dump another table in hbase at the time i feel very boring. so i was planing to do this action in manually. so i planned to write some code in java. it will like to share the code with you guys.
                                 
                                  The Flow chart of my coding
My codes:
 Mainclass.java
 package com.srini.myexperiment;
 import java.sql.SQLException;
 import java.util.List;
 import net.neoremind.sshxcute.exception.TaskExecFailException;
 import org.apache.commons.mail.EmailException;
 import org.apache.log4j.Logger;

public class MainClass {
final static Logger logger = Logger.getLogger(MainClass.class);

public static void main(String[] args) throws SQLException,
TaskExecFailException, EmailException {
logger.info("Entering application.");

RunTheSqoopCommnad ru = new RunTheSqoopCommnad();
List<String> table = ru.gettable();
ru.runCommand(table);
logger.info("Exiting application.");
}

}

GetTableInfo.java
package com.srini.myexperiment;
import java.sql.Connection;
import java.sql.DatabaseMetaData;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.util.ArrayList;
import java.util.List;

import org.apache.commons.dbcp.BasicDataSource;
import org.apache.commons.mail.EmailException;
import org.apache.log4j.Logger;

public class GetTableInfo {
final  Logger logger = Logger.getLogger(this.getClass());
public List<String> gettable() throws EmailException {
logger .info("Mysql Datbase Connectivity is start");
BasicDataSource n = new BasicDataSource();
n.setDriverClassName("com.mysql.jdbc.Driver");
n.setUrl("jdbc:mysql://xxx.xxx.xxx.xxx:3306/vms");
n.setUsername("root");
n.setPassword("root");
Connection conn = null;
try {
conn = n.getConnection();
} catch (SQLException e) {
          logger .error("Database Connection error",e);
          logger .warn("this warning message");
// TODO Auto-generated catch block
          GetMail maill=new GetMail();
          maill.getmessage(e.getMessage());
}
DatabaseMetaData dbm = null;
try {
dbm = conn.getMetaData();
} catch (SQLException e) {
// TODO Auto-generated catch block
logger .error("it can't get the metadat information",e);
logger .warn("this warning message");
GetMail maill=new GetMail();
         maill.getmessage(e.getMessage());
}
String[] types = { "TABLE" };
ResultSet re = null;
try {
re = dbm.getTables(null, null, "%", types);
} catch (SQLException e) {
// TODO Auto-generated catch block
logger .error("The tabel not found error",e);
logger .warn("this warning message");
GetMail maill=new GetMail();
         maill.getmessage(e.getMessage());
}
List<String> result = new ArrayList<String>();
try {
while (re.next()) {
String ta = re.getString("TABLE_NAME");
result.add(ta);
}
} catch (SQLException e) {
// TODO Auto-generated catch block
logger .error("it can't retrive the table in the database",e);
logger .warn("this warning message");
GetMail maill=new GetMail();
         maill.getmessage(e.getMessage());
}
return result;

}

}
RunTheSqoopCommnad.java
package com.srini.myexperiment;

import java.util.List;

import org.apache.commons.mail.EmailException;
import org.apache.log4j.Logger;

import net.neoremind.sshxcute.core.ConnBean;
import net.neoremind.sshxcute.core.SSHExec;
import net.neoremind.sshxcute.exception.TaskExecFailException;
import net.neoremind.sshxcute.task.CustomTask;
import net.neoremind.sshxcute.task.impl.ExecCommand;

public class RunTheSqoopCommnad extends GetTableInfo {
final  Logger logger = Logger.getLogger(this.getClass());
public void runCommand(List<String> tableinfo) throws EmailException {
logger .info("Mysql Datbase Connectivity is start");
String command = "create 'vms',";
for (int j = 0; j < tableinfo.size(); j++) {
command = command.concat(" " + "\'" + tableinfo.get(j) + "\'" + ","
+ " ");
}
command = command.substring(0, command.length() - 2);
ConnBean conn = new ConnBean("192.168.119.131", "cloudera", "cloudera");
SSHExec ssh = SSHExec.getInstance(conn);
ssh.connect();
logger .error("ssh connect connect");
CustomTask hbase = new ExecCommand("echo" + " " + "\"" + command + "\""
+ " | hbase shell");
try {
ssh.exec(hbase);
} catch (TaskExecFailException e) {
// TODO Auto-generated catch block
logger .error("it can't able to create the hbase table ",e);
logger .warn("This Is the warning");
GetMail maill=new GetMail();
         maill.getmessage(e.getMessage());
         e.printStackTrace();
}
CustomTask hdfs = new ExecCommand("su hdfs");
try {
ssh.exec(hdfs);
} catch (TaskExecFailException e) {
// TODO Auto-generated catch block
logger .error("sudo user error ",e);
logger .warn("This Is the warning");
GetMail maill=new GetMail();
         maill.getmessage(e.getMessage());
}
for (int i = 0; i < tableinfo.size(); i++) {
CustomTask cu = new ExecCommand(
"sqoop import -Dsqoop.hbase.add.row.key=true --connect jdbc:mysql://xxx.xxx.xxx.xxx:3306/vms --username root --password root --table"
+ " "
+ tableinfo.get(i)
+ " "
+ "--hbase-table vms --column-family"
+ " "
+ tableinfo.get(i));
try {
ssh.exec(cu);
} catch (TaskExecFailException e) {
// TODO Auto-generated catch block
logger .error("sqoop command err ",e);
logger .warn("This Is the warning");
GetMail maill=new GetMail();
         maill.getmessage(e.getMessage());
         e.printStackTrace();
}

}
ssh.disconnect();
GetMail ma=new GetMail();
ma.getmessage("The Query has been executed successfully in you server");
}

}
GetMail.java
package com.srini.myexperiment;

import org.apache.commons.mail.EmailAttachment;
import org.apache.commons.mail.EmailException;
import org.apache.commons.mail.MultiPartEmail;

public class GetMail {
public void getmessage(String message) throws EmailException {
EmailAttachment emaii = new EmailAttachment();
emaii.setPath("D:\\loging.log");
emaii.setDisposition(EmailAttachment.ATTACHMENT);
emaii.setDescription("logging information");
emaii.setName("Srinivasan");
MultiPartEmail mail = new MultiPartEmail();
mail.setHostName("smtp.googlemail.com");
mail.setSmtpPort(465);
mail.setAuthentication("eceseenu1989", "password");
mail.setSSLOnConnect(true);
mail.setFrom("eceseenu1989@gmail.com");
mail.setSubject("test mail");
mail.setMsg(message);
mail.addTo("eceseenu19898.2@gmail.com");
mail.attach(emaii);
mail.send();
}

}
log4j.properties
# Log levels
log4j.rootLogger=INFO,CONSOLE,R
# Appender Configuration
log4j.appender.CONSOLE=org.apache.log4j.ConsoleAppender
# Pattern to output the caller's file name and line number
log4j.appender.CONSOLE.layout=org.apache.log4j.PatternLayout
log4j.appender.CONSOLE.layout.ConversionPattern=%5p [%t] (%F:%L) - %m%n
# Rolling File Appender
log4j.appender.R=org.apache.log4j.RollingFileAppender
# Path and file name to store the log file
log4j.appender.R.File=D:\\loging.log
log4j.appender.R.MaxFileSize=200KB
# Number of backup files
log4j.appender.R.MaxBackupIndex=2
# Layout for Rolling File Appender
log4j.appender.R.layout=org.apache.log4j.PatternLayout
log4j.appender.R.layout.ConversionPattern=%d - %c - %p - %m%n