tpc-ds 性能测试
1.安装依赖
yum -y install gcc gcc-c++ libstdc++-devel bison byacc flex
2.下载
http://www.tpc.org/tpc_documents_current_versions/current_specifications.asp
3安装
unzip tpc-ds-tool.zip
cd v2.3.0/tools
make
4.生成数据
生成100G数据
./dsdgen -scale 100 -dir /tmp/data
并行生产数据
dbgen2–scale 100 –dir /tmp –parallel 4 –child 1 &
dbgen2–scale 100 –dir /tmp –parallel 4 –child 2 &
dbgen2–scale 100 –dir /tmp –parallel 4 –child 3 &
dbgen2–scale 100 –dir /tmp –parallel 4 –child 4 &
更新
dbgen2 –scale 100 –dir /tmp –update 3
生成数据中：
最后生成的数据：（大约4个小时）
5、将本地数据上传到 HDFS
[root@d02 ~]# hadoop fs -mkdir /tpc_ds
[root@d02 ~]# hadoop fs -put /tmp/data/* /tpc_ds
[root@d02 ~]# hadoop fs -mkdir /tpc_ds
[root@d02 ~]# hadoop fs -put /tmp/data/* /tpc_ds
6、在 SQL 引擎中创建外表和执行表，导入数据
在hive中建表
drop table if exists call_center;
create external table call_center(
cc_call_center_sk bigint
, cc_call_center_id string
, cc_rec_start_date string
, cc_rec_end_date string
, cc_closed_date_sk bigint
, cc_open_date_sk bigint
, cc_name string
, cc_class string
, cc_employees int
, cc_sq_ft int
, cc_hours string
, cc_manager string
, cc_mkt_id int
, cc_mkt_class string
, cc_mkt_desc string
, cc_market_manager string
, cc_division int
, cc_division_name string
, cc_company int
, cc_company_name string
, cc_street_number string
, cc_street_name string
, cc_street_type string
, cc_suite_number string
, cc_city string
, cc_county string
, cc_state string
, cc_zip string
, cc_country string
, cc_gmt_offset double
, cc_tax_percentage double
)
row format delimited fields terminated by '|'
STORED AS textfile;
drop table if exists catalog_page;
create external table catalog_page(
cp_catalog_page_sk bigint
, cp_catalog_page_id string
, cp_start_date_sk bigint
, cp_end_date_sk bigint
, cp_department string
, cp_catalog_number int
, cp_catalog_page_number int
, cp_description string
, cp_type string
)
row format delimited fields terminated by '|'
STORED AS textfile;
drop table if exists catalog_returns;
create external table catalog_returns
(
cr_returned_date_sk bigint,
cr_returned_time_sk bigint,
cr_item_sk bigint,
cr_refunded_customer_sk bigint,
cr_refunded_cdemo_sk bigint,
cr_refunded_hdemo_sk bigint,
cr_refunded_addr_sk bigint,
cr_returning_customer_sk bigint,
cr_returning_cdemo_sk bigint,
cr_returning_hdemo_sk bigint,
cr_returning_addr_sk bigint,
cr_call_center_sk bigint,
cr_catalog_page_sk bigint,
cr_ship_mode_sk bigint,
cr_warehouse_sk bigint,
cr_reason_sk bigint,
cr_order_number bigint,
cr_return_quantity int,
cr_return_amount double,
cr_return_tax double,
cr_return_amt_inc_tax double,
cr_fee double,
cr_return_ship_cost double,
cr_refunded_cash double,
cr_reversed_charge double,
cr_store_credit double,
cr_net_loss double
)
row format delimited fields terminated by '|'
STORED AS textfile;
drop table if exists catalog_sales;
create external table catalog_sales
(
cs_sold_date_sk bigint,
cs_sold_time_sk bigint,
cs_ship_date_sk bigint,
cs_bill_customer_sk bigint,
cs_bill_cdemo_sk bigint,
cs_bill_hdemo_sk bigint,
cs_bill_addr_sk bigint,
cs_ship_customer_sk bigint,
cs_ship_cdemo_sk bigint,
cs_ship_hdemo_sk bigint,
cs_ship_addr_sk bigint,
cs_call_center_sk bigint,
cs_catalog_page_sk bigint,
cs_ship_mode_sk bigint,
cs_warehouse_sk bigint,
cs_item_sk bigint,
cs_promo_sk bigint,
cs_order_number bigint,
cs_quantity int,
cs_wholesale_cost double,
cs_list_price double,
cs_sales_price double,
cs_ext_discount_amt double,
cs_ext_sales_price double,
cs_ext_wholesale_cost double,
cs_ext_list_price double,
cs_ext_tax double,
cs_coupon_amt double,
cs_ext_ship_cost double,
cs_net_paid double,
cs_net_paid_inc_tax double,
cs_net_paid_inc_ship double,
cs_net_paid_inc_ship_tax double,
cs_net_profit double
)
row format delimited fields terminated by '|'
STORED AS textfile;
drop table if exists customer_address;
create external table customer_address
(
ca_address_sk bigint,
ca_address_id string,
ca_street_number string,
ca_street_name string,
ca_street_type string,
ca_suite_number string,
ca_city string,
ca_county string,
ca_state string,
ca_zip string,
ca_country string,
ca_gmt_offset double,
ca_location_type string
)
row format delimited fields terminated by '|'
STORED AS textfile;
drop table if exists customer_demographics;
create external table customer_demographics
(
cd_demo_sk bigint,
cd_gender string,
cd_marital_status string,
cd_education_status string,
cd_purchase_estimate int,
cd_credit_rating string,
cd_dep_count int,
cd_dep_employed_count int,
cd_dep_college_count int
)
row format delimited fields terminated by '|'
STORED AS textfile;
drop table if exists customer;
create external table customer
(
c_customer_sk bigint,
c_customer_id string,
c_current_cdemo_sk bigint,
c_current_hdemo_sk bigint,
c_current_addr_sk bigint,
c_first_shipto_date_sk bigint,
c_first_sales_date_sk bigint,
c_salutation string,
c_first_name string,
c_last_name string,
c_preferred_cust_flag string,
c_birth_day int,
c_birth_month int,
c_birth_year int,
c_birth_country string,
c_login string,
c_email_address string,
c_last_review_date string
)
row format delimited fields terminated by '|'
STORED AS textfile;
drop table if exists date_dim;
create external table date_dim
(
d_date_sk bigint,
d_date_id string,
d_date string,
d_month_seq int,
d_week_seq int,
d_quarter_seq int,
d_year int,
d_dow int,
d_moy int,
d_dom int,
d_qoy int,
d_fy_year int,
d_fy_quarter_seq int,
d_fy_week_seq int,
d_day_name string,
d_quarter_name string,
d_holiday string,
d_weekend string,
d_following_holiday string,
d_first_dom int,
d_last_dom int,
d_same_day_ly int,
d_same_day_lq int,
d_current_day string,
d_current_week string,
d_current_month string,
d_current_quarter string,
d_current_year string
)
row format delimited fields terminated by '|'
STORED AS textfile;
drop table if exists household_demographics;
create external table household_demographics
(
hd_demo_sk bigint,
hd_income_band_sk bigint,
hd_buy_potential string,
hd_dep_count int,
hd_vehicle_count int
)
row format delimited fields terminated by '|'
STORED AS textfile;
drop table if exists income_band;
create external table income_band(
ib_income_band_sk bigint
, ib_lower_bound int
, ib_upper_bound int
)
row format delimited fields terminated by '|'
STORED AS textfile;
drop table if exists inventory;
create external table inventory
(
inv_date_sk bigint,
inv_item_sk bigint,
inv_warehouse_sk bigint,
inv_quantity_on_hand int
)
row format delimited fields terminated by '|'
STORED AS textfile;
drop table if exists item;
create external table item
(
i_item_sk bigint,
i_item_id string,
i_rec_start_date string,
i_rec_end_date string,
i_item_desc string,
i_current_price double,
i_wholesale_cost double,
i_brand_id int,
i_brand string,
i_class_id int,
i_class string,
i_category_id int,
i_category string,
i_manufact_id int,
i_manufact string,
i_size string,
i_formulation string,
i_color string,
i_units string,
i_container string,
i_manager_id int,
i_product_name string
)
row format delimited fields terminated by '|'
STORED AS textfile;
drop table if exists promotion;
create external table promotion
(
p_promo_sk bigint,
p_promo_id string,
p_start_date_sk bigint,
p_end_date_sk bigint,
p_item_sk bigint,
p_cost double,
p_response_target int,
p_promo_name string,
p_channel_dmail string,
p_channel_email string,
p_channel_catalog string,
p_channel_tv string,
p_channel_radio string,
p_channel_press string,
p_channel_event string,
p_channel_demo string,
p_channel_details string,
p_purpose string,
p_discount_active string
)
row format delimited fields terminated by '|'
STORED AS textfile;
drop table if exists reason;
create external table reason(
r_reason_sk bigint
, r_reason_id string
, r_reason_desc string
)
row format delimited fields terminated by '|'
STORED AS textfile;
drop table if exists ship_mode;
create external table ship_mode(
sm_ship_mode_sk bigint
, sm_ship_mode_id string
, sm_type string
, sm_code string
, sm_carrier string
, sm_contract string
)
row format delimited fields terminated by '|'
STORED AS textfile;
drop table if exists store_returns;
create external table store_returns
(
sr_returned_date_sk bigint,
sr_return_time_sk bigint,
sr_item_sk bigint,
sr_customer_sk bigint,
sr_cdemo_sk bigint,
sr_hdemo_sk bigint,
sr_addr_sk bigint,
sr_store_sk bigint,
sr_reason_sk bigint,
sr_ticket_number bigint,
sr_return_quantity int,
sr_return_amt double,
sr_return_tax double,
sr_return_amt_inc_tax double,
sr_fee double,
sr_return_ship_cost double,
sr_refunded_cash double,
sr_reversed_charge double,
sr_store_credit double,
sr_net_loss double
)
row format delimited fields terminated by '|'
STORED AS textfile;
drop table if exists store_sales;
create external table store_sales
(
ss_sold_date_sk bigint,
ss_sold_time_sk bigint,
ss_item_sk bigint,
ss_customer_sk bigint,
ss_cdemo_sk bigint,
ss_hdemo_sk bigint,
ss_addr_sk bigint,
ss_store_sk bigint,
ss_promo_sk bigint,
ss_ticket_number bigint,
ss_quantity int,
ss_wholesale_cost double,
ss_list_price double,
ss_sales_price double,
ss_ext_discount_amt double,
ss_ext_sales_price double,
ss_ext_wholesale_cost double,
ss_ext_list_price double,
ss_ext_tax double,
ss_coupon_amt double,
ss_net_paid double,
ss_net_paid_inc_tax double,
ss_net_profit double
)
row format delimited fields terminated by '|'
STORED AS textfile;
drop table if exists store;
create external table store
(
s_store_sk bigint,
s_store_id string,
s_rec_start_date string,
s_rec_end_date string,
s_closed_date_sk bigint,
s_store_name string,
s_number_employees int,
s_floor_space int,
s_hours string,
s_manager string,
s_market_id int,
s_geography_class string,
s_market_desc string,
s_market_manager string,
s_division_id int,
s_division_name string,
s_company_id int,
s_company_name string,
s_street_number string,
s_street_name string,
s_street_type string,
s_suite_number string,
s_city string,
s_county string,
s_state string,
s_zip string,
s_country string,
s_gmt_offset double,
s_tax_precentage double
)
row format delimited fields terminated by '|'
STORED AS textfile;
drop table if exists time_dim;
create external table time_dim
(
t_time_sk bigint,
t_time_id string,
t_time int,
t_hour int,
t_minute int,
t_second int,
t_am_pm string,
t_shift string,
t_sub_shift string,
t_meal_time string
)
row format delimited fields terminated by '|'
STORED AS textfile;
drop table if exists warehouse;
create external table warehouse(
w_warehouse_sk bigint
, w_warehouse_id string
, w_warehouse_name string
, w_warehouse_sq_ft int
, w_street_number string
, w_street_name string
, w_street_type string
, w_suite_number string
, w_city string
, w_county string
, w_state string
, w_zip string
, w_country string
, w_gmt_offset double
)
row format delimited fields terminated by '|'
STORED AS textfile;
drop table if exists web_page;
create external table web_page(
wp_web_page_sk bigint
, wp_web_page_id string
, wp_rec_start_date string
, wp_rec_end_date string
, wp_creation_date_sk bigint
, wp_access_date_sk bigint
, wp_autogen_flag string
, wp_customer_sk bigint
, wp_url string
, wp_type string
, wp_char_count int
, wp_link_count int
, wp_image_count int
, wp_max_ad_count int
)
row format delimited fields terminated by '|'
STORED AS textfile;
drop table if exists web_returns;
create external table web_returns
(
wr_returned_date_sk bigint,
wr_returned_time_sk bigint,
wr_item_sk bigint,
wr_refunded_customer_sk bigint,
wr_refunded_cdemo_sk bigint,
wr_refunded_hdemo_sk bigint,
wr_refunded_addr_sk bigint,
wr_returning_customer_sk bigint,
wr_returning_cdemo_sk bigint,
wr_returning_hdemo_sk bigint,
wr_returning_addr_sk bigint,
wr_web_page_sk bigint,
wr_reason_sk bigint,
wr_order_number bigint,
wr_return_quantity int,
wr_return_amt double,
wr_return_tax double,
wr_return_amt_inc_tax double,
wr_fee double,
wr_return_ship_cost double,
wr_refunded_cash double,
wr_reversed_charge double,
wr_account_credit double,
wr_net_loss double
)
row format delimited fields terminated by '|'
STORED AS textfile;
drop table if exists web_sales;
create external table web_sales
(
ws_sold_date_sk bigint,
ws_sold_time_sk bigint,
ws_ship_date_sk bigint,
ws_item_sk bigint,
ws_bill_customer_sk bigint,
ws_bill_cdemo_sk bigint,
ws_bill_hdemo_sk bigint,
ws_bill_addr_sk bigint,
ws_ship_customer_sk bigint,
ws_ship_cdemo_sk bigint,
ws_ship_hdemo_sk bigint,
ws_ship_addr_sk bigint,
ws_web_page_sk bigint,
ws_web_site_sk bigint,
ws_ship_mode_sk bigint,
ws_warehouse_sk bigint,
ws_promo_sk bigint,
ws_order_number bigint,
ws_quantity int,
ws_wholesale_cost double,
ws_list_price double,
ws_sales_price double,
ws_ext_discount_amt double,
ws_ext_sales_price double,
ws_ext_wholesale_cost double,
ws_ext_list_price double,
ws_ext_tax double,
ws_coupon_amt double,
ws_ext_ship_cost double,
ws_net_paid double,
ws_net_paid_inc_tax double,
ws_net_paid_inc_ship double,
ws_net_paid_inc_ship_tax double,
ws_net_profit double
)
row format delimited fields terminated by '|'
STORED AS textfile;
drop table if exists web_site;
create external table web_site
(
web_site_sk bigint,
web_site_id string,
web_rec_start_date string,
web_rec_end_date string,
web_name string,
web_open_date_sk bigint,
web_close_date_sk bigint,
web_class string,
web_manager string,
web_mkt_id int,
web_mkt_class string,
web_mkt_desc string,
web_market_manager string,
web_company_id int,
web_company_name string,
web_street_number string,
web_street_name string,
web_street_type string,
web_suite_number string,
web_city string,
web_county string,
web_state string,
web_zip string,
web_country string,
web_gmt_offset double,
web_tax_percentage double
)
row format delimited fields terminated by '|'
STORED AS textfile;在hive中建表
drop table if exists call_center;
create external table call_center(
cc_call_center_sk bigint
, cc_call_center_id string
, cc_rec_start_date string
, cc_rec_end_date string
, cc_closed_date_sk bigint
, cc_open_date_sk bigint
, cc_name string
, cc_class string
, cc_employees int
, cc_sq_ft int
, cc_hours string
, cc_manager string
, cc_mkt_id int
, cc_mkt_class string
, cc_mkt_desc string
, cc_market_manager string
, cc_division int
, cc_division_name string
, cc_company int
, cc_company_name string
, cc_street_number string
, cc_street_name string
, cc_street_type string
, cc_suite_number string
, cc_city string
, cc_county string
, cc_state string
, cc_zip string
, cc_country string
, cc_gmt_offset double
, cc_tax_percentage double
)
row format delimited fields terminated by '|'
STORED AS textfile;
drop table if exists catalog_page;
create external table catalog_page(
cp_catalog_page_sk bigint
, cp_catalog_page_id string
, cp_start_date_sk bigint
, cp_end_date_sk bigint
, cp_department string
, cp_catalog_number int
, cp_catalog_page_number int
, cp_description string
, cp_type string
)
row format delimited fields terminated by '|'
STORED AS textfile;
drop table if exists catalog_returns;
create external table catalog_returns
(
cr_returned_date_sk bigint,
cr_returned_time_sk bigint,
cr_item_sk bigint,
cr_refunded_customer_sk bigint,
cr_refunded_cdemo_sk bigint,
cr_refunded_hdemo_sk bigint,
cr_refunded_addr_sk bigint,
cr_returning_customer_sk bigint,
cr_returning_cdemo_sk bigint,
cr_returning_hdemo_sk bigint,
cr_returning_addr_sk bigint,
cr_call_center_sk bigint,
cr_catalog_page_sk bigint,
cr_ship_mode_sk bigint,
cr_warehouse_sk bigint,
cr_reason_sk bigint,
cr_order_number bigint,
cr_return_quantity int,
cr_return_amount double,
cr_return_tax double,
cr_return_amt_inc_tax double,
cr_fee double,
cr_return_ship_cost double,
cr_refunded_cash double,
cr_reversed_charge double,
cr_store_credit double,
cr_net_loss double
)
row format delimited fields terminated by '|'
STORED AS textfile;
drop table if exists catalog_sales;
create external table catalog_sales
(
cs_sold_date_sk bigint,
cs_sold_time_sk bigint,
cs_ship_date_sk bigint,
cs_bill_customer_sk bigint,
cs_bill_cdemo_sk bigint,
cs_bill_hdemo_sk bigint,
cs_bill_addr_sk bigint,
cs_ship_customer_sk bigint,
cs_ship_cdemo_sk bigint,
cs_ship_hdemo_sk bigint,
cs_ship_addr_sk bigint,
cs_call_center_sk bigint,
cs_catalog_page_sk bigint,
cs_ship_mode_sk bigint,
cs_warehouse_sk bigint,
cs_item_sk bigint,
cs_promo_sk bigint,
cs_order_number bigint,
cs_quantity int,
cs_wholesale_cost double,
cs_list_price double,
cs_sales_price double,
cs_ext_discount_amt double,
cs_ext_sales_price double,
cs_ext_wholesale_cost double,
cs_ext_list_price double,
cs_ext_tax double,
cs_coupon_amt double,
cs_ext_ship_cost double,
cs_net_paid double,
cs_net_paid_inc_tax double,
cs_net_paid_inc_ship double,
cs_net_paid_inc_ship_tax double,
cs_net_profit double
)
row format delimited fields terminated by '|'
STORED AS textfile;
drop table if exists customer_address;
create external table customer_address
(
ca_address_sk bigint,
ca_address_id string,
ca_street_number string,
ca_street_name string,
ca_street_type string,
ca_suite_number string,
ca_city string,
ca_county string,
ca_state string,
ca_zip string,
ca_country string,
ca_gmt_offset double,
ca_location_type string
)
row format delimited fields terminated by '|'
STORED AS textfile;
drop table if exists customer_demographics;
create external table customer_demographics
(
cd_demo_sk bigint,
cd_gender string,
cd_marital_status string,
cd_education_status string,
cd_purchase_estimate int,
cd_credit_rating string,
cd_dep_count int,
cd_dep_employed_count int,
cd_dep_college_count int
)
row format delimited fields terminated by '|'
STORED AS textfile;
drop table if exists customer;
create external table customer
(
c_customer_sk bigint,
c_customer_id string,
c_current_cdemo_sk bigint,
c_current_hdemo_sk bigint,
c_current_addr_sk bigint,
c_first_shipto_date_sk bigint,
c_first_sales_date_sk bigint,
c_salutation string,
c_first_name string,
c_last_name string,
c_preferred_cust_flag string,
c_birth_day int,
c_birth_month int,
c_birth_year int,
c_birth_country string,
c_login string,
c_email_address string,
c_last_review_date string
)
row format delimited fields terminated by '|'
STORED AS textfile;
drop table if exists date_dim;
create external table date_dim
(
d_date_sk bigint,
d_date_id string,
d_date string,
d_month_seq int,
d_week_seq int,
d_quarter_seq int,
d_year int,
d_dow int,
d_moy int,
d_dom int,
d_qoy int,
d_fy_year int,
d_fy_quarter_seq int,
d_fy_week_seq int,
d_day_name string,
d_quarter_name string,
d_holiday string,
d_weekend string,
d_following_holiday string,
d_first_dom int,
d_last_dom int,
d_same_day_ly int,
d_same_day_lq int,
d_current_day string,
d_current_week string,
d_current_month string,
d_current_quarter string,
d_current_year string
)
row format delimited fields terminated by '|'
STORED AS textfile;
drop table if exists household_demographics;
create external table household_demographics
(
hd_demo_sk bigint,
hd_income_band_sk bigint,
hd_buy_potential string,
hd_dep_count int,
hd_vehicle_count int
)
row format delimited fields terminated by '|'
STORED AS textfile;
drop table if exists income_band;
create external table income_band(
ib_income_band_sk bigint
, ib_lower_bound int
, ib_upper_bound int
)
row format delimited fields terminated by '|'
STORED AS textfile;
drop table if exists inventory;
create external table inventory
(
inv_date_sk bigint,
inv_item_sk bigint,
inv_warehouse_sk bigint,
inv_quantity_on_hand int
)
row format delimited fields terminated by '|'
STORED AS textfile;
drop table if exists item;
create external table item
(
i_item_sk bigint,
i_item_id string,
i_rec_start_date string,
i_rec_end_date string,
i_item_desc string,
i_current_price double,
i_wholesale_cost double,
i_brand_id int,
i_brand string,
i_class_id int,
i_class string,
i_category_id int,
i_category string,
i_manufact_id int,
i_manufact string,
i_size string,
i_formulation string,
i_color string,
i_units string,
i_container string,
i_manager_id int,
i_product_name string
)
row format delimited fields terminated by '|'
STORED AS textfile;
drop table if exists promotion;
create external table promotion
(
p_promo_sk bigint,
p_promo_id string,
p_start_date_sk bigint,
p_end_date_sk bigint,
p_item_sk bigint,
p_cost double,
p_response_target int,
p_promo_name string,
p_channel_dmail string,
p_channel_email string,
p_channel_catalog string,
p_channel_tv string,
p_channel_radio string,
p_channel_press string,
p_channel_event string,
p_channel_demo string,
p_channel_details string,
p_purpose string,
p_discount_active string
)
row format delimited fields terminated by '|'
STORED AS textfile;
drop table if exists reason;
create external table reason(
r_reason_sk bigint
, r_reason_id string
, r_reason_desc string
)
row format delimited fields terminated by '|'
STORED AS textfile;
drop table if exists ship_mode;
create external table ship_mode(
sm_ship_mode_sk bigint
, sm_ship_mode_id string
, sm_type string
, sm_code string
, sm_carrier string
, sm_contract string
)
row format delimited fields terminated by '|'
STORED AS textfile;
drop table if exists store_returns;
create external table store_returns
(
sr_returned_date_sk bigint,
sr_return_time_sk bigint,
sr_item_sk bigint,
sr_customer_sk bigint,
sr_cdemo_sk bigint,
sr_hdemo_sk bigint,
sr_addr_sk bigint,
sr_store_sk bigint,
sr_reason_sk bigint,
sr_ticket_number bigint,
sr_return_quantity int,
sr_return_amt double,
sr_return_tax double,
sr_return_amt_inc_tax double,
sr_fee double,
sr_return_ship_cost double,
sr_refunded_cash double,
sr_reversed_charge double,
sr_store_credit double,
sr_net_loss double
)
row format delimited fields terminated by '|'
STORED AS textfile;
drop table if exists store_sales;
create external table store_sales
(
ss_sold_date_sk bigint,
ss_sold_time_sk bigint,
ss_item_sk bigint,
ss_customer_sk bigint,
ss_cdemo_sk bigint,
ss_hdemo_sk bigint,
ss_addr_sk bigint,
ss_store_sk bigint,
ss_promo_sk bigint,
ss_ticket_number bigint,
ss_quantity int,
ss_wholesale_cost double,
ss_list_price double,
ss_sales_price double,
ss_ext_discount_amt double,
ss_ext_sales_price double,
ss_ext_wholesale_cost double,
ss_ext_list_price double,
ss_ext_tax double,
ss_coupon_amt double,
ss_net_paid double,
ss_net_paid_inc_tax double,
ss_net_profit double
)
row format delimited fields terminated by '|'
STORED AS textfile;
drop table if exists store;
create external table store
(
s_store_sk bigint,
s_store_id string,
s_rec_start_date string,
s_rec_end_date string,
s_closed_date_sk bigint,
s_store_name string,
s_number_employees int,
s_floor_space int,
s_hours string,
s_manager string,
s_market_id int,
s_geography_class string,
s_market_desc string,
s_market_manager string,
s_division_id int,
s_division_name string,
s_company_id int,
s_company_name string,
s_street_number string,
s_street_name string,
s_street_type string,
s_suite_number string,
s_city string,
s_county string,
s_state string,
s_zip string,
s_country string,
s_gmt_offset double,
s_tax_precentage double
)
row format delimited fields terminated by '|'
STORED AS textfile;
drop table if exists time_dim;
create external table time_dim
(
t_time_sk bigint,
t_time_id string,
t_time int,
t_hour int,
t_minute int,
t_second int,
t_am_pm string,
t_shift string,
t_sub_shift string,
t_meal_time string
)
row format delimited fields terminated by '|'
STORED AS textfile;
drop table if exists warehouse;
create external table warehouse(
w_warehouse_sk bigint
, w_warehouse_id string
, w_warehouse_name string
, w_warehouse_sq_ft int
, w_street_number string
, w_street_name string
, w_street_type string
, w_suite_number string
, w_city string
, w_county string
, w_state string
, w_zip string
, w_country string
, w_gmt_offset double
)
row format delimited fields terminated by '|'
STORED AS textfile;
drop table if exists web_page;
create external table web_page(
wp_web_page_sk bigint
, wp_web_page_id string
, wp_rec_start_date string
, wp_rec_end_date string
, wp_creation_date_sk bigint
, wp_access_date_sk bigint
, wp_autogen_flag string
, wp_customer_sk bigint
, wp_url string
, wp_type string
, wp_char_count int
, wp_link_count int
, wp_image_count int
, wp_max_ad_count int
)
row format delimited fields terminated by '|'
STORED AS textfile;
drop table if exists web_returns;
create external table web_returns
(
wr_returned_date_sk bigint,
wr_returned_time_sk bigint,
wr_item_sk bigint,
wr_refunded_customer_sk bigint,
wr_refunded_cdemo_sk bigint,
wr_refunded_hdemo_sk bigint,
wr_refunded_addr_sk bigint,
wr_returning_customer_sk bigint,
wr_returning_cdemo_sk bigint,
wr_returning_hdemo_sk bigint,
wr_returning_addr_sk bigint,
wr_web_page_sk bigint,
wr_reason_sk bigint,
wr_order_number bigint,
wr_return_quantity int,
wr_return_amt double,
wr_return_tax double,
wr_return_amt_inc_tax double,
wr_fee double,
wr_return_ship_cost double,
wr_refunded_cash double,
wr_reversed_charge double,
wr_account_credit double,
wr_net_loss double
)
row format delimited fields terminated by '|'
STORED AS textfile;
drop table if exists web_sales;
create external table web_sales
(
ws_sold_date_sk bigint,
ws_sold_time_sk bigint,
ws_ship_date_sk bigint,
ws_item_sk bigint,
ws_bill_customer_sk bigint,
ws_bill_cdemo_sk bigint,
ws_bill_hdemo_sk bigint,
ws_bill_addr_sk bigint,
ws_ship_customer_sk bigint,
ws_ship_cdemo_sk bigint,
ws_ship_hdemo_sk bigint,
ws_ship_addr_sk bigint,
ws_web_page_sk bigint,
ws_web_site_sk bigint,
ws_ship_mode_sk bigint,
ws_warehouse_sk bigint,
ws_promo_sk bigint,
ws_order_number bigint,
ws_quantity int,
ws_wholesale_cost double,
ws_list_price double,
ws_sales_price double,
ws_ext_discount_amt double,
ws_ext_sales_price double,
ws_ext_wholesale_cost double,
ws_ext_list_price double,
ws_ext_tax double,
ws_coupon_amt double,
ws_ext_ship_cost double,
ws_net_paid double,
ws_net_paid_inc_tax double,
ws_net_paid_inc_ship double,
ws_net_paid_inc_ship_tax double,
ws_net_profit double
)
row format delimited fields terminated by '|'
STORED AS textfile;
drop table if exists web_site;
create external table web_site
(
web_site_sk bigint,
web_site_id string,
web_rec_start_date string,
web_rec_end_date string,
web_name string,
web_open_date_sk bigint,
web_close_date_sk bigint,
web_class string,
web_manager string,
web_mkt_id int,
web_mkt_class string,
web_mkt_desc string,
web_market_manager string,
web_company_id int,
web_company_name string,
web_street_number string,
web_street_name string,
web_street_type string,
web_suite_number string,
web_city string,
web_county string,
web_state string,
web_zip string,
web_country string,
web_gmt_offset double,
web_tax_percentage double
)
row format delimited fields terminated by '|'
STORED AS textfile;
7、加载数据到hive里
LOAD DATA inpath '/tpc_ds/data/call_center.dat' INTO TABLE call_center;
LOAD DATA inpath '/tpc_ds/data/catalog_page.dat' INTO TABLE catalog_page;
LOAD DATA inpath '/tpc_ds/data/catalog_returns.dat' INTO TABLE catalog_returns;
LOAD DATA inpath '/tpc_ds/data/catalog_sales.dat' INTO TABLE catalog_sales;
LOAD DATA inpath '/tpc_ds/data/customer.dat' INTO TABLE customer;
LOAD DATA inpath '/tpc_ds/data/customer_address.dat' INTO TABLE customer_address;
LOAD DATA inpath '/tpc_ds/data/customer_demographics.dat' INTO TABLE customer_demographics;
LOAD DATA inpath '/tpc_ds/data/date_dim.dat' INTO TABLE date_dim;
LOAD DATA inpath '/tpc_ds/data/household_demographics.dat' INTO TABLE household_demographics;
LOAD DATA inpath '/tpc_ds/data/income_band.dat' INTO TABLE income_band;
LOAD DATA inpath '/tpc_ds/data/inventory.dat' INTO TABLE inventory;
LOAD DATA inpath '/tpc_ds/data/item.dat' INTO TABLE item;
LOAD DATA inpath '/tpc_ds/data/promotion.dat' INTO TABLE promotion;
LOAD DATA inpath '/tpc_ds/data/reason.dat' INTO TABLE reason;
LOAD DATA inpath '/tpc_ds/data/ship_mode.dat' INTO TABLE ship_mode;
LOAD DATA inpath '/tpc_ds/data/store.dat' INTO TABLE store;
LOAD DATA inpath '/tpc_ds/data/store_returns.dat' INTO TABLE store_returns;
LOAD DATA inpath '/tpc_ds/data/store_sales.dat' INTO TABLE store_sales;
LOAD DATA inpath '/tpc_ds/data/time_dim.dat' INTO TABLE time_dim;
LOAD DATA inpath '/tpc_ds/data/warehouse.dat' INTO TABLE warehouse;
LOAD DATA inpath '/tpc_ds/data/web_sales.dat' INTO TABLE web_sales;
LOAD DATA inpath '/tpc_ds/data/web_page.dat' INTO TABLE web_page;
LOAD DATA inpath '/tpc_ds/data/web_returns.dat' INTO TABLE web_returns;
LOAD DATA inpath '/tpc_ds/data/web_site.dat' INTO TABLE web_site;
INSERT OVERWRITE TABLE call_center SELECT * FROM tpc_ps.call_center;
INSERT OVERWRITE TABLE catalog_page SELECT * FROM tpc_ps.catalog_page;
INSERT OVERWRITE TABLE catalog_returns SELECT * FROM tpc_ps.catalog_returns;
INSERT OVERWRITE TABLE catalog_sales SELECT * FROM tpc_ps.catalog_sales;
INSERT OVERWRITE TABLE customer SELECT * FROM tpc_ps.customer;
INSERT OVERWRITE TABLE customer_address SELECT * FROM tpc_ps.customer_address;
INSERT OVERWRITE TABLE customer_demographics SELECT * FROM tpc_ps.customer_demographics;
INSERT OVERWRITE TABLE date_dim SELECT * FROM tpc_ps.date_dim;
INSERT OVERWRITE TABLE dbgen_version SELECT * FROM tpc_ps.dbgen_version;
INSERT OVERWRITE TABLE household_demographics SELECT * FROM tpc_ps.household_demographics;
INSERT OVERWRITE TABLE income_band SELECT * FROM tpc_ps.income_band;
INSERT OVERWRITE TABLE inventory SELECT * FROM tpc_ps.inventory;
INSERT OVERWRITE TABLE item SELECT * FROM tpc_ps.item;
INSERT OVERWRITE TABLE promotion SELECT * FROM tpc_ps.promotion;
INSERT OVERWRITE TABLE reason SELECT * FROM tpc_ps.reason;
INSERT OVERWRITE TABLE ship_mode SELECT * FROM tpc_ps.ship_mode;
INSERT OVERWRITE TABLE store SELECT * FROM tpc_ps.store;
INSERT OVERWRITE TABLE store_returns SELECT * FROM tpc_ps.store_returns;
INSERT OVERWRITE TABLE store_sales SELECT * FROM tpc_ps.store_sales;
INSERT OVERWRITE TABLE time_dim SELECT * FROM tpc_ps.time_dim;
INSERT OVERWRITE TABLE warehouse SELECT * FROM tpc_ps.warehouse;
INSERT OVERWRITE TABLE web_page SELECT * FROM tpc_ps.web_page;
INSERT OVERWRITE TABLE web_returns SELECT * FROM tpc_ps.web_returns;
INSERT OVERWRITE TABLE web_sales SELECT * FROM tpc_ps.web_sales;
INSERT OVERWRITE TABLE web_site SELECT * FROM tpc_ps.web_site;
8、执行 Linux 清理缓存操作
sync
echo 3 > /proc/sys/vm/drop_caches
9、执行测试sql