Skip to content

L
label
提交 2df226da 编写于 作者: wuyunfeng's avatar wuyunfeng
浏览文件
操作

添加两个字段module_class1,module_class2

上级 d1b8a427
展开全部 隐藏空白字符变更
内嵌 并排
正在显示 包含 509 行增加 和 29 行删除
src/main/scala/com/config/MyConfigSession.scala
浏览文件 @ 2df226da
......@@ -14,6 +14,7 @@ object MyConfigSession {
final val HIVE_TABLE1: String = "pica_dw.dw_fact_log_session"
final val HIVE_TABLE2: String = "pica_dw.dw_fact_log_session_path"
final val HIVE_TABLE3: String = "pica_dw.dw_fact_log_session_heart"
final val HIVE_TABLE4: String = "pica_dw.dw_fact_log_session_pref"
//写入的文件路径
final val PARQUET_PATH: String = "hdfs://bi-name1:8020/tmp/output/"
......@@ -34,7 +35,16 @@ object MyConfigSession {
final val ACTION_CATEGORY_SQL_HEART: String = "select action_type,action_category from pica_dw.dw_dim_log_action_type where action_type='ACTION_HEART_BEAT'"
final val ACTION_URLLABEL_SQL:String = "select url_content,label_value from pica_dw.dw_dim_log_action_urllabel "
//从源表pica_log.picalog_trace_app_part中执行SQL获取源数据,这里获取昨天的
final val SOURCE_SQL_PREF: String =
"""
|select device_token,pseudo_session,class_name,action,view_path,component_tag,created,mobile,doctor_id,device_brand,device_model,app_version,
|device_type,web_data,web_data_type,alternate_info,network_type,remark1 login_state,remark2 first_app_version,user_token_tourist,serviceName
| from pica_log.picalog_trace_app_part
| where pseudo_session is not null and pseudo_session !=''
| and pseudo_id !='' and extra_info !='com.picahealth.patient' and serviceName != 'trace3'
| and FROM_UNIXTIME(cast(substring(created,1,10) as bigint),'yyyy-MM-dd')=created_day and created is not null and created!=''
""".stripMargin
//从源表pica_log.picalog_trace_app_part中执行SQL获取源数据,这里获取昨天的
final val SOURCE_SQL: String =
"""
......@@ -61,7 +71,7 @@ object MyConfigSession {
final val SOURCE_SQL_PATH: String =
s"""
|select id log_session_id, session_id,user_id_int user_id,action_type,user_token,menu_code,action_code,position,label_value,label_class,
|app_version,device_type,created_time,date_time from ${MyConfigSession.HIVE_TABLE1}
|app_version,device_type,created_time,date_time,module_class1,module_class2 from ${MyConfigSession.HIVE_TABLE1}
| where app_version >= '3.1.7'
| AND ((action_type ='ACTION_CLICK' and action_code != 'null' ) OR action_type ='ACTION_VIEW' )
| and (menu_code != '0' and menu_code !='null' and menu_code !='' and length(menu_code) <= 3 )
......@@ -75,20 +85,40 @@ object MyConfigSession {
s"""
|SELECT t.session_id, COALESCE(cast(b.id as string),'0') AS user_id, t.mobile, t.device_token, t.user_token,
|t.view_class,t.view_path,t.action_type,t.component_tag, t.menu_code,
|t.action_code, t.position, t.label_value,t.label_class,t.app_version,t.device_type,
|t.action_code, t.position, t.label_value,t.label_class,t.module_class1,t.module_class2,t.app_version,t.device_type,
|t.device_brand, t.device_model, t.device_system,t.net_type,t.created_time,
|t.date_time from ${MyConfigSession.VIEW_SESSION_ODS} as t
|left join pica_ds.pica_doctor as b on t.user_id = cast(b.id as string)
""".stripMargin
final val INIT_USER_ID_SQL_PREF =
s"""
|SELECT t.pseudo_session, COALESCE(cast(b.id as string),'0') AS user_id, t.mobile, t.device_token, t.user_token,
|t.view_class,t.view_path,t.action, t.component_tag, t.menu_code,
|t.action_code, t.position, t.label_value,t.label_class,t.app_version,t.device_type,
|t.device_brand, t.device_model, t.net_type,t.created_time,t.date_time,
|t.web_data,t.web_data_type,t.alternate_info,t.login_state,t.first_app_version,t.serviceName
| from ${MyConfigSession.VIEW_SESSION_ODS} as t
|left join pica_ds.pica_doctor as b on t.user_id = cast(b.id as string)
""".stripMargin
//1.针对没有匹配到的user_id,先使用 mobile_phone 进行匹配,得到 user_id 匹配,'0'
final val MOBILE_PHONE_SQL: String =
s"""
|SELECT ss.session_id, COALESCE(cast(b.id as string),'0') AS user_id, ss.mobile, ss.device_token, ss.user_token,
|ss.view_class,ss.view_path,ss.action_type,ss.component_tag, ss.menu_code,
|ss.action_code, ss.position,ss.label_value,ss.label_class,ss.app_version, ss.device_type,
|ss.device_brand, ss.device_model,ss.device_system,ss.net_type,ss.created_time,
|ss.action_code, ss.position,ss.label_value,ss.label_class,ss.module_class1,ss.module_class2,ss.app_version, ss.device_type,
|ss.device_brand, ss.device_model,ss.device_system, ss.net_type,ss.created_time,
|ss.date_time from ${MyConfigSession.VIEW_SESSION_NO_MATCH} AS ss
|left join (select distinct id,mobile_phone from pica_ds.pica_doctor where pica_doctor.delete_flag = 1 ) AS b on ss.mobile = b.mobile_phone
""".stripMargin
final val MOBILE_PHONE_SQL_PREF: String =
s"""
|SELECT ss.pseudo_session,COALESCE(cast(b.id as string),'0') AS user_id,ss.mobile,ss.device_token,ss.user_token,
|ss.view_class,ss.view_path,ss.action, ss.component_tag,ss.menu_code,
|ss.action_code,ss.position,ss.label_value,ss.label_class,ss.app_version,ss.device_type,
|ss.device_brand,ss.device_model, ss.net_type,ss.created_time,
|ss.date_time,ss.web_data,ss.web_data_type,ss.alternate_info,ss.login_state,ss.first_app_version,ss.serviceName
| from ${MyConfigSession.VIEW_SESSION_NO_MATCH} AS ss
|left join (select distinct id,mobile_phone from pica_ds.pica_doctor where pica_doctor.delete_flag = 1 ) AS b on ss.mobile = b.mobile_phone
""".stripMargin
//2.使用equipment表匹配,默认是昨天的
final val EQUIPMENT_INFO_SQL: String =
......@@ -112,12 +142,21 @@ object MyConfigSession {
s"""
|SELECT t.session_id, COALESCE(cast(b.user_id as string),'0') AS user_id, t.mobile, t.device_token, t.user_token,
|t.view_class,t.view_path,t.action_type,t.component_tag, t.menu_code,
|t.action_code, t.position, t.label_value,t.label_class,t.app_version,t.device_type,
|t.action_code, t.position, t.label_value,t.label_class,t.module_class1,t.module_class2,t.app_version,t.device_type,
|t.device_brand, t.device_model, t.device_system,t.net_type,t.created_time,
|t.date_time from (select * from ${MyConfigSession.VIEW_MOBILE_PHONE} a where a.user_id= '0' ) as t
|left join ${MyConfigSession.VIEW_EQUIPMENT_INFO} as b on t.device_token = b.device_token
""".stripMargin
final val DEVICE_TOKEN_SQL_PREF: String =
s"""
|SELECT t.pseudo_session,COALESCE(cast(b.user_id as string),'0') AS user_id,t.mobile,t.device_token,t.user_token,
|t.view_class,t.view_path,t.action ,t.component_tag,t.menu_code,
|t.action_code,t.position,t.label_value,t.label_class,t.app_version,t.device_type,
|t.device_brand,t.device_model, t.net_type,t.created_time,
|t.date_time,t.web_data,t.web_data_type,t.alternate_info,t.login_state,t.first_app_version,t.serviceName
| from (select * from ${MyConfigSession.VIEW_MOBILE_PHONE} a where a.user_id= '0' ) as t
|left join ${MyConfigSession.VIEW_EQUIPMENT_INFO} as b on t.device_token = b.device_token
""".stripMargin
......
src/main/scala/com/session/SessionProcessPath.scala
浏览文件 @ 2df226da
......@@ -121,9 +121,9 @@ object SessionProcessPath {
StringUtils.getNotNullString(row.getAs[String]("device_type")),
StringUtils.getNotNullString(row.getAs[String]("created_time")),
StringUtils.getNotNullString(row.getAs[String]("date_time")),
StringUtils.getNotNullString(row.getAs[String]("module_class1")),
StringUtils.getNotNullString(row.getAs[String]("module_class2")),
row.getAs[Double]("refer_time_diff")
// StringUtils.getNotNullString(row.getAs[String]("module_class1")),
// StringUtils.getNotNullString(row.getAs[String]("module_class2"))
))
})
rowList.iterator
......@@ -150,6 +150,7 @@ object SessionProcessPath {
}catch {
case e:Exception => {
println("-----------------------------------任务异常---------------------------------------------------")
e.printStackTrace()
val exceptionSQL: String =
s"""
|update ${MyConfigSession.JDBC_TABLE} set status=?,exception=?,end_time=? where job_id=1968 and start_time='${startTime}'
......@@ -194,7 +195,7 @@ object SessionProcessPath {
"COALESCE(refer_position,'') as refer_position",
"COALESCE(refer_action_type,'') as refer_action_type",
"COALESCE(refer_created,created_time) as refer_created",
"step_id", "app_version", "device_type", "created_time", "date_time")
"step_id", "app_version", "device_type", "created_time", "date_time","module_class1","module_class2")
//在此基础上增加字段 refer_time_diff,值为 created_time, refer_created 之差
val referTimeDiff: DataFrame =
......@@ -219,7 +220,7 @@ object SessionProcessPath {
| select log_session_id, session_id,user_id,action_type,user_token,menu_code,action_code,position,label_value,label_class,
| refer_menu_code,refer_action_code,refer_position,refer_action_type,
| cast(refer_time_diff as int) as refer_time_diff,
| step_id,app_version,device_type,created_time,date_time,'' module_class1,'' module_class2
| step_id,app_version,device_type,created_time,date_time, module_class1, module_class2
| from result_view distribute by rand()
""".stripMargin
sparkSession.sql(loadDataSql)
......
src/main/scala/com/session/SessionProcessPref.scala 0 → 100644
浏览文件 @ 2df226da
此差异已折叠。
src/main/scala/com/utils/Driver.scala
浏览文件 @ 2df226da
package com.utils
import com.session.{SessionMenuCalc, SessionProcess, SessionProcessHeart, SessionProcessPath }
import com.session.{SessionMenuCalc, SessionProcess, SessionProcessHeart, SessionProcessPath, SessionProcessPref}
import org.apache.hadoop.util.ProgramDriver
/**
......@@ -15,6 +15,7 @@ object Driver {
driver.addClass("SessionProcessPath",classOf[SessionProcessPath],"用户Session数据分析导入到dw_fact_log_session_path表")
driver.addClass("SessionProcessHeart",classOf[SessionProcessHeart],"用户Session数据分析导入到dw_fact_log_session_heart表")
driver.addClass("SessionMenuCalc",classOf[SessionMenuCalc],"传递日期参数--用户Session数据分析导入到dw_fact_log_session_menu_calc表")
driver.addClass("SessionProcessPref",classOf[SessionProcessPref],"传递日期参数--用户Session数据etl导入到dw_fact_log_sesson_pref表")
driver.run(args)
}
}
Markdown 格式
0% or
您添加了 0 人 到此讨论。请谨慎行事。
先完成此消息的编辑！
想要评论请 注册 或