hadoop hbase 相关代码运行问题

2017-11-08

###HBase代码在hadoop集群下运行找不到hbase相关类

hbase代码引入的jar都可以在hbase/lib的目录下找到，一次性都导入，代码就能运行了。单纯的hbase代码是可以直接作为普通java程序运行的，不用再hadoop集群下运行。

但是hbase相关的读写写到了MapReduce的操作中，就需要打成jar包，在maven下，将hbase/lib下的jar包都导入项目，然后代码就不会报错了，然后rebuild project 完成后，就可以通过mvn compile,mvn package将代码打包了。

通过hadoop jar Your.jar classname [hdfs:inputdir] [hdfs:outputdir]时会报hbase中的相关类找不到，原因是hadoop集群找不到本地的hbase jar包。

很容易理解就是hadoop无法找到对应hbase相关的jar包。试了好多方法，也历经了好几天无法运行的情况，反而优先完成了Hive实验。最后的解决办法是在\$HADOOP_HOME/etc/hadoop/hadoop-env.sh中对应有个添加HADOOP_CLASSPATH的地方，在前面多加一句

export HADOOP_CLASSPATH = \$HADOOP_CLASSPATH:/home/hadoop/hadoop_installs/hbase-1.2.6/lib/*.jar

可以通过hadoop classpath命令查看hadoop集群内所有jar包的路径

并且在yarn-site.xml中添加相关的jar包所在的位置，也就是增加一个property如下所示

<property>
			<name>yarn.application.classpath</name>
			<value>
			/home/hadoop/hadoop_installs/hadoop-2.7.4/share/hadoop/common/*,
			/home/hadoop/hadoop_installs/hadoop-2.7.4/share/hadoop/common/lib/*,
			/home/hadoop/hadoop_installs/hadoop-2.7.4/share/hadoop/mapreduce/*,
			/home/hadoop/hadoop_installs/hadoop-2.7.4/share/hadoop/mapreduce/lib/*,
			/home/hadoop/hadoop_installs/hadoop-2.7.4/share/hadoop/hdfs/*,
			/home/hadoop/hadoop_installs/hadoop-2.7.4/share/hadoop/hdfs/lib/*,
			/home/hadoop/hadoop_installs/hadoop-2.7.4/share/hadoop/yarn/*,
			/home/hadoop/hadoop_installs/hadoop-2.7.4/share/hadoop/yarn/lib/*,
			/home/hadoop/hadoop_installs/hbase-1.2.6/lib/*
			</value>
	</property>

HBase Java代码的基本实现

/**
* Created by ganjun on 2017/11/4.
*/
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.HBaseConfiguration;
import org.apache.hadoop.hbase.HColumnDescriptor;
import org.apache.hadoop.hbase.HTableDescriptor;
import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.client.Delete;
import org.apache.hadoop.hbase.client.HBaseAdmin;
import org.apache.hadoop.hbase.client.HTable;
import org.apache.hadoop.hbase.client.Put;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.hbase.client.Scan;
import org.apache.hadoop.hbase.client.ResultScanner;
import org.apache.hadoop.hbase.client.Result;
import org.apache.hadoop.hbase.KeyValue;
import java.io.FileWriter;
import java.io.IOException;
public class HBaseManage {
    private static Configuration conf = null;
    static
    {
        conf = HBaseConfiguration.create();
//        conf.set("hbase.zookeeper.quorum", "nodel");
//        conf.set("hbase.zookeeper.property.clientPort", "2181");
    }
    public static void createTable(String tableName, String[] familys)
            throws Exception{
        HBaseAdmin admin = new HBaseAdmin(conf);
        if (admin.tableExists(tableName)){
            System.out.println("table "+tableName+" already exists!");
        }else{
            HTableDescriptor tableDesc = new HTableDescriptor(TableName.valueOf(tableName));
            for (int i=0 ; i<familys.length ; i++){
                tableDesc.addFamily(new HColumnDescriptor(familys[i]));
            }
            admin.createTable(tableDesc);
            System.out.println("create table "+tableName+" success!");
        }
    }
    //表数据添加
    public static void addData(String tableName, String rowKey, String family,
                               String qualifier, String value) throws Exception{
        try{
            HTable table = new HTable(conf , tableName);
            Put put = new Put(Bytes.toBytes(rowKey));
            put.add(Bytes.toBytes(family), Bytes.toBytes(qualifier), Bytes.toBytes(value));
            table.put(put);
            System.out.println("insert recored success!");
        }catch (IOException ex){
            ex.printStackTrace();
        }
    }
    //表数据删除一行
    public static void deleteRow(String tableName, String rowKey)
            throws IOException {
        HTable table = new HTable(conf, tableName);
        Delete deleteRow = new Delete(rowKey.getBytes());
        table.delete(deleteRow);
        System.out.println("delete row "+rowKey+" success!");
    }
    //表数据删除某一列
    public static void deleteColumn(String tableName, String rowKey, String falilyName, String columnName)
            throws IOException {
        HTable table = new HTable(conf , tableName);
        Delete deleteColumn = new Delete(Bytes.toBytes(rowKey));
        deleteColumn.deleteColumn(Bytes.toBytes(falilyName), Bytes.toBytes(columnName));
        table.delete(deleteColumn);
        System.out.println("delete " + rowKey + ":" + falilyName + " : " + columnName + " success");
    }
    //保存一个表的数据到某一文件中
    public static void tableToFile(String tableName, String filename)
            throws IOException{
        FileWriter out = new FileWriter(filename);
        HTable table = new HTable(conf, tableName);
        Scan s = new Scan();
        s.setMaxVersions();
        ResultScanner ss= table.getScanner(s);
        for(Result r:ss){
            System.out.println(new String(r.getRow()));
            for(KeyValue kv:r.raw()){
                System.out.println(new String(kv.getValue()));
                out.write(new String(r.getRow(), "utf-8") + "\t" + new String(kv.getValue()) + "\n");
            }
        }
        out.close();
    }
    public static void main(String [] args){
        try{
            String tableName = "Wuxia";
            String[] familys = {"Count"};
         //   HBaseManage.createTable(tableName, familys);
            HBaseManage.addData(tableName, "江湖", "Count", "NormalCount", "3.0");
            HBaseManage.addData(tableName, "风雪", "Count", "NormalCount", "1.0");
            HBaseManage.tableToFile(tableName, "wuxia.txt");
        }catch (Exception ex){
            ex.printStackTrace();
        }
    }
}