提交 bb76e55c 编写于 作者: wuyunfeng's avatar wuyunfeng

修改sessionpref表名

上级 5c56dda7
...@@ -9,7 +9,7 @@ package com.config ...@@ -9,7 +9,7 @@ package com.config
*/ */
object MyConfigSession { object MyConfigSession {
//Hive的DW层流量表 //Hive的DW层流量表
final val HIVE_TABLE0: String = "pica_dw.dw_fact_log_session_pref" final val HIVE_TABLE0: String = "pica_ods.ods_log_session_pref"
final val HIVE_TABLE1_TMP: String = "pica_dw.dw_fact_log_session_tmp" final val HIVE_TABLE1_TMP: String = "pica_dw.dw_fact_log_session_tmp"
final val HIVE_TABLE2_TMP: String = "pica_dw.dw_fact_log_session_path_tmp" final val HIVE_TABLE2_TMP: String = "pica_dw.dw_fact_log_session_path_tmp"
final val HIVE_TABLE1: String = "pica_dw.dw_fact_log_session" final val HIVE_TABLE1: String = "pica_dw.dw_fact_log_session"
......
...@@ -54,6 +54,7 @@ object SessionProcessPref { ...@@ -54,6 +54,7 @@ object SessionProcessPref {
var index = 0 var index = 0
val conditionGroup = List("<='4' ","between '5' and '9'",">'9'") val conditionGroup = List("<='4' ","between '5' and '9'",">'9'")
for(condition <- conditionGroup){ for(condition <- conditionGroup){
index += 1
val slideDF = sourceDF.where(s" SUBSTRING(pseudo_session,1,1) ${condition}").repartition(100) val slideDF = sourceDF.where(s" SUBSTRING(pseudo_session,1,1) ${condition}").repartition(100)
println(s"-----------------------------------compute refer columns,condition=${condition}-----------------------------------------") println(s"-----------------------------------compute refer columns,condition=${condition}-----------------------------------------")
//step2:抽取出当天pseudo_session对应的非空的device_token,doctor_id,mobile,补充到对应的pseudo_session下这几项为空的记录中 //step2:抽取出当天pseudo_session对应的非空的device_token,doctor_id,mobile,补充到对应的pseudo_session下这几项为空的记录中
...@@ -243,7 +244,7 @@ class SessionProcessPref extends java.io.Serializable { ...@@ -243,7 +244,7 @@ class SessionProcessPref extends java.io.Serializable {
if(index!=1){ if(index!=1){
insertSql = "insert into" insertSql = "insert into"
} }
println(s"-----------------create view ${tmpTable} and load to dw_fact_log_session_pref--------------------") println(s"-----------------create view ${tmpTable} and load to ${MyConfigSession.HIVE_TABLE0} --------------------")
dataFrame.repartition(10).createOrReplaceTempView(tmpTable) dataFrame.repartition(10).createOrReplaceTempView(tmpTable)
val fields = List("pseudo_session", "user_id", "COALESCE(cast(user_id as int),0) user_id_int", "mobile", "device_token", "user_token", "view_class", "view_path", "action", "action_type", val fields = List("pseudo_session", "user_id", "COALESCE(cast(user_id as int),0) user_id_int", "mobile", "device_token", "user_token", "view_class", "view_path", "action", "action_type",
"component_tag", "menu_code", "menu_code_new", "action_code", "position", "label_value", "label_class", "module_class1", "module_class2", "app_version", "device_type", "device_brand", "component_tag", "menu_code", "menu_code_new", "action_code", "position", "label_value", "label_class", "module_class1", "module_class2", "app_version", "device_type", "device_brand",
......
...@@ -46,7 +46,7 @@ object SessionProcessTerm { ...@@ -46,7 +46,7 @@ object SessionProcessTerm {
var condition = " 1=1" var condition = " 1=1"
if (args.length >= 1) { if (args.length >= 1) {
scnData = args(0) scnData = args(0)
if(args.length > 1){ if(args.length > 1 && args(1)!=""){
condition = args(1) condition = args(1)
} }
} }
...@@ -90,9 +90,9 @@ object SessionProcessTerm { ...@@ -90,9 +90,9 @@ object SessionProcessTerm {
println("selectDF.show========") println("selectDF.show========")
selectDF.printSchema() selectDF.printSchema()
println("selectDF.count=========",selectDF.count()) println("selectDF.count=========",selectDF.count())
// val conditionGroup = List("<='4' ","between '5' and '9'",">'9'") val conditionGroup = List("<='4' ","between '5' and '9'",">'9'")
val conditionGroup = List("='0'","='1'","='2'","='3'","='4'","='5'","='6'","='7'","='8'","='9'", // val conditionGroup = List("='0'","='1'","='2'","='3'","='4'","='5'","='6'","='7'","='8'","='9'",
"='a'","='b'","='c'","='d'","='e'","='f'") // "='a'","='b'","='c'","='d'","='e'","='f'")
var dataCount = 0 var dataCount = 0
var index = 0 var index = 0
selectDF.persist(StorageLevel.MEMORY_AND_DISK_SER) selectDF.persist(StorageLevel.MEMORY_AND_DISK_SER)
......
Markdown 格式
0% or
您添加了 0 到此讨论。请谨慎行事。
先完成此消息的编辑!
想要评论请 注册