/**
*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.hbase.client;
import java.io.IOException;
import java.lang.reflect.Constructor;
import java.util.concurrent.ExecutorService;
import org.apache.hadoop.hbase.classification.InterfaceAudience;
import org.apache.hadoop.hbase.classification.InterfaceStability;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.HBaseConfiguration;
import org.apache.hadoop.hbase.security.User;
import org.apache.hadoop.hbase.security.UserProvider;
/**
* A non-instantiable class that manages creation of {@link Connection}s.
* Managing the lifecycle of the {@link Connection}s to the cluster is the responsibility of
* the caller.
* From a {@link Connection}, {@link Table} implementations are retrieved
* with {@link Connection#getTable(TableName)}. Example:
*
* Connection connection = ConnectionFactory.createConnection(config);
* Table table = connection.getTable(TableName.valueOf("table1"));
* try {
* // Use the table as needed, for a single operation and a single thread
* } finally {
* table.close();
* connection.close();
* }
*
*
* Similarly, {@link Connection} also returns {@link Admin} and {@link RegionLocator}
* implementations.
*
* This class replaces {@link HConnectionManager}, which is now deprecated.
* @see Connection
* @since 0.99.0
*/
@InterfaceAudience.Public
@InterfaceStability.Evolving
public class ConnectionFactory {
/** No public c.tors */
protected ConnectionFactory() {
}
/**
* Create a new Connection instance using default HBaseConfiguration. Connection
* encapsulates all housekeeping for a connection to the cluster. All tables and interfaces
* created from returned connection share zookeeper connection, meta cache, and connections
* to region servers and masters.
*
* The caller is responsible for calling {@link Connection#close()} on the returned
* connection instance.
*
* Typical usage:
*
*
* @return Connection object for conf
*/
public static Connection createConnection() throws IOException {
return createConnection(HBaseConfiguration.create(), null, null);
}
/**
* Create a new Connection instance using the passed conf instance. Connection
* encapsulates all housekeeping for a connection to the cluster. All tables and interfaces
* created from returned connection share zookeeper connection, meta cache, and connections
* to region servers and masters.
*
* The caller is responsible for calling {@link Connection#close()} on the returned
* connection instance.
*
* Typical usage:
*
*
* @param conf configuration
* @return Connection object for conf
*/
public static Connection createConnection(Configuration conf) throws IOException {
return createConnection(conf, null, null);
}
/**
* Create a new Connection instance using the passed conf instance. Connection
* encapsulates all housekeeping for a connection to the cluster. All tables and interfaces
* created from returned connection share zookeeper connection, meta cache, and connections
* to region servers and masters.
*
* The caller is responsible for calling {@link Connection#close()} on the returned
* connection instance.
*
* Typical usage:
*
*
* @param conf configuration
* @param pool the thread pool to use for batch operations
* @return Connection object for conf
*/
public static Connection createConnection(Configuration conf, ExecutorService pool)
throws IOException {
return createConnection(conf, pool, null);
}
/**
* Create a new Connection instance using the passed conf instance. Connection
* encapsulates all housekeeping for a connection to the cluster. All tables and interfaces
* created from returned connection share zookeeper connection, meta cache, and connections
* to region servers and masters.
*
* The caller is responsible for calling {@link Connection#close()} on the returned
* connection instance.
*
* Typical usage:
*
*
* @param conf configuration
* @param user the user the connection is for
* @return Connection object for conf
*/
public static Connection createConnection(Configuration conf, User user)
throws IOException {
return createConnection(conf, null, user);
}
/**
* Create a new Connection instance using the passed conf instance. Connection
* encapsulates all housekeeping for a connection to the cluster. All tables and interfaces
* created from returned connection share zookeeper connection, meta cache, and connections
* to region servers and masters.
*
* The caller is responsible for calling {@link Connection#close()} on the returned
* connection instance.
*
* Typical usage:
*
*
* @param conf configuration
* @param user the user the connection is for
* @param pool the thread pool to use for batch operations
* @return Connection object for conf
*/
public static Connection createConnection(Configuration conf, ExecutorService pool, User user)
throws IOException {
if (user == null) {
UserProvider provider = UserProvider.instantiate(conf);
user = provider.getCurrent();
}
return createConnection(conf, false, pool, user);
}
static Connection createConnection(final Configuration conf, final boolean managed,
final ExecutorService pool, final User user)
throws IOException {
String className = conf.get(HConnection.HBASE_CLIENT_CONNECTION_IMPL,
ConnectionManager.HConnectionImplementation.class.getName());
Class clazz = null;
try {
clazz = Class.forName(className);
} catch (ClassNotFoundException e) {
throw new IOException(e);
}
try {
// Default HCM#HCI is not accessible; make it so before invoking.
Constructor constructor =
clazz.getDeclaredConstructor(Configuration.class,
boolean.class, ExecutorService.class, User.class);
constructor.setAccessible(true);
return (Connection) constructor.newInstance(conf, managed, pool, user);
} catch (Exception e) {
throw new IOException(e);
}
}
}
一：安装mysql 下载安装包wget http://dev.mysql.com/get/mysql-community-release-el7-5.noarch.rpm 解压：rpm -ivh mysql-community-release-el7-5.noarch.rpm 安装yum install mysql-community-server 重启mysql服务：service mysqld restart mysql -u root 为root用户设置密码root：mysql> set password for ‘root’@‘localhost’ =password(‘root’); 配置文件/etc/my.cnf加上编码配置：[mysql] default-character-set =utf8 grant all privileges on . to root@’ %'identified by ‘root’; flush privileges; 刷新权限
javax.jdo.option.ConnectionURLjdbc:mysql://127.0.0.1:3306/hive?characterEncoding=UTF-8&serverTimezone=GMT%2B8
JDBC connect string for a JDBC metastore.
To use SSL to encrypt/authenticate the connection, provide database-specific SSL flag in the connection URL.
For example, jdbc:postgresql://myhost/db?ssl=true for postgres database.
javax.jdo.option.ConnectionDriverNamecom.mysql.cj.jdbc.DriverDriver class name for a JDBC metastorejavax.jdo.option.ConnectionUserNamerootUsername to use against metastore databasejavax.jdo.option.ConnectionPassword123456password to use against metastore databasehive.exec.local.scratchdir/usr/local/hive/apache-hive-2.3.4-bin/tmp/${user.name}Local scratch space for Hive jobshive.downloaded.resources.dir/usr/local/hive/apache-hive-2.3.4-bin/iotmp/${hive.session.id}_resourcesTemporary local directory for added resources in the remote file system.hive.querylog.location/usr/local/hive/apache-hive-2.3.4-bin/iotmp/${system:user.name}Location of Hive run time structured log filehive.server2.logging.operation.log.location/usr/local/hive/apache-hive-2.3.4-bin/iotmp/${system:user.name}/operation_logsTop level directory where operation logs are stored if logging functionality is enabledhive.server2.thrift.bind.hostbigdataBind host on which to run the HiveServer2 Thrift service.system:java.io.tmpdir/usr/local/hive/apache-hive-2.3.4-bin/iotmp
1、create table words(line string);
2、load data inpath '/input/1.txt' overwrite into table words;
3、create table wordcount as select word, count(1) as count from (select explode(split(line,' '))as word from words) w group by word order by word;
4、select * from wordcount;
import org.apache.spark.SparkContext
import org.apache.spark.SparkContext._
import org.apache.spark.SparkConf
object WordCount {
def main(args: Array[String]) {
val inputFile = "file:///usr/local/spark/mycode/wordcount/word.txt"
val conf = new SparkConf().setAppName("WordCount").setMaster("local[2]")
val sc = new SparkContext(conf)
val textFile = sc.textFile(inputFile)
val wordCount = textFile.flatMap(line => line.split(" ")).map(word => (word, 1)).reduceByKey((a, b) => a + b)
wordCount.foreach(println)
}
}
七、java程序实现wordcount统计 程序详情：
import org.apache.spark.SparkConf;
import org.apache.spark.api.java.JavaPairRDD;
import org.apache.spark.api.java.JavaRDD;
import org.apache.spark.api.java.JavaSparkContext;
import scala.Tuple2;
import java.util.Arrays;
public class JavaWordCount {
public static void main(String[] args) {
SparkConf conf = new SparkConf().setAppName("Spark WordCount written by java!");
JavaSparkContext sc = new JavaSparkContext(conf);
JavaRDD textFile = sc.textFile("hdfs:///user/hadoop/word.txt");
JavaPairRDD counts = textFile
.flatMap(s -> Arrays.asList(s.split(" ")).iterator())
.mapToPair(word -> new Tuple2<>(word, 1))
.reduceByKey((a, b) -> a + b);
counts.saveAsTextFile(hdfs:///user/hadoop/writeback");
sc.close();
}
}
This article is from an interview with Zuhaib Siddique, a production engineer at HipChat, makers of group chat and IM for teams.
HipChat started in an unusual space, one you might not
