|ss.date_time from ${MyConfigSession.VIEW_SESSION_NO_MATCH} AS ss
|left join (select distinct id,mobile_phone from pica_ds.pica_doctor where pica_doctor.delete_flag = 1 ) AS b on ss.mobile = b.mobile_phone
|left join (select distinct id,mobile_phone from pica_ds.pica_doctor where pica_doctor.delete_flag = 1 and mobile_phone!='' and mobile_phone!='XK0HdMN6dAfOlYPOFHHL0A==' ) AS b on ss.mobile = b.mobile_phone
""".stripMargin
finalvalMOBILE_PHONE_SQL_PREF:String=
s"""
...
...
@@ -125,8 +134,9 @@ object MyConfigSession {
finalvalEQUIPMENT_INFO_SQL:String=
"""
|SELECT a.user_id,a.device_token ,ROW_NUMBER() OVER ( PARTITION BY a.device_token ORDER BY a.creat_time DESC ) row_d
|from pica_ds.picams_equipment_info AS a
| where a.user_id IS NOT NULL
|from (select e1.user_id,e1.device_token,e1.creat_time,e1.modify_time,e1.delete_flag from pica_ds.picams_equipment_info e1 union
| select e2.user_id,e2.device_token,e2.creat_time, e2.modify_time,e2.delete_flag from pica_ds.picams_p_equipment_info_shadow e2 ) a
| where a.user_id IS NOT NULL and a.user_id>0 and a.delete_flag=1 and device_token is not null
| AND (to_date(a.creat_time) = date_sub(current_date(),1) OR to_date(a.modify_time) = date_sub(current_date(),1))
valfields=List("pseudo_session","user_id","COALESCE(cast(user_id as int),0) user_id_int","mobile","device_token","user_token","view_class","view_path","action","action_type",
valloadDataSql=s"${insertSql} table ${MyConfigSession.HIVE_TABLE0} partition(created_day='${partitionDay}') select ${fields.mkString(",")} from ${tmpTable} distribute by rand()"
valloadDataSql=s"${insertSql} table ${MyConfigSession.HIVE_TABLE0} partition(created_day='${partitionDay}') select concat(regexp_replace( '${partitionDay}','-','') ,cast( (row_number() over(partition by 1 order by created_time) +${count}) as string)) as id,"+
s"${fields.mkString(",")} from ${tmpTable} distribute by rand()"
sparkSession.sql(loadDataSql)
}
}
...
...
@@ -271,8 +265,8 @@ case class SessionPref(pseudo_session: String,
| where servicename='trace2' and action!='ACTION_EQUIP_INFO'
| and (case when ((view_class like '%YunqueApp%' and action!='ACTION_HEART_BEAT') or LENGTH(view_class)<=3 or view_class='YQGuidePageViewVC') then '2' else '1' end)='1'
|select b.* from ( ${SOURCE_SQL_COMMON + " and serviceName='trace1' "}) b where b.doctor_id in (
| select doctor_id
| from (${SOURCE_SQL_COMMON +" and doctor_id not in ('','0') and doctor_id is not null" }) s group by doctor_id having concat_ws( '_',collect_set(service_name)) ='trace1'
| )
| and b.pseudo_session not in (select pseudo_session from source_add_table group by pseudo_session )
|select concat(regexp_replace( '${scnData}',"-","") ,cast(row_number() over(partition by 1 order by a.created_time) as string)) as id,
|select
|a.session_id,
|a.device_token,
|a.user_id,
|a.user_token,
|a.mobile,
|a.menu_code,
|a.menu_begin_time,
| case when a.action_type in('ACTION_VIEW','ACTION_HEART') and b.menu_end_time is null then c.session_end_time else b.menu_end_time end menu_end_time,
|(cast((case when a.action_type in('ACTION_VIEW','ACTION_HEART') and b.menu_end_time is null then c.session_end_time else b.menu_end_time end) as bigint)-cast(a.menu_begin_time as bigint))/1000 menu_time_diff,
|a.action_type,
|a.action_code,
|a.position,
|a.label_value,
|lag(a.menu_code) over(partition by a.session_id order by a.created_time) refer_menu_code,
|lag(a.action_code) over(partition by a.session_id order by a.created_time) refer_action_code,
|lag(a.position) over(partition by a.session_id order by a.created_time) refer_position,
|lag(a.label_value) over(partition by a.session_id order by a.created_time) refer_label_value,
|lag(a.action_type) over(partition by a.session_id order by a.created_time) refer_action_type,
|a.label_class,
|a.module_class1,
|a.module_class2,
|a.action_type,
|a.menu_begin_time,
| case when a.action_type in('ACTION_VIEW','ACTION_HEART') and b.menu_end_time is null then c.session_end_time else b.menu_end_time end menu_end_time,
|(cast((case when a.action_type in('ACTION_VIEW','ACTION_HEART') and b.menu_end_time is null then c.session_end_time else b.menu_end_time end) as bigint)-cast(a.menu_begin_time as bigint))/1000 menu_time_diff,
|a.action_step,
|a.device_type,
|a.device_brand,
|a.device_model,
|a.app_version,
|a.net_type,
|a.created_time,
|a.date_time,
|c.session_begin_time,
|c.session_end_time,
|(cast(c.session_end_time as bigint)-cast(c.session_begin_time as bigint))/1000 session_time_diff,
|a.refer_session_id
|a.pre_session_id,
|case when a.user_id=0 then a.device_token else a.user_id end user_identity_id,
|left join refer_menu_table b on a.session_id=b.session_id and a.device_token=b.device_token and a.user_id=b.user_id and a.menu_code=b.menu_code and a.created_time=b.created_time and a.action_type in('ACTION_VIEW','ACTION_HEART')
|left join session_end_table c on a.session_id = c.session_id
|SELECT t.*,t.user_id_ods user_id_old, COALESCE(cast(b.id as string),'0') AS user_id from ${MyConfigSession.VIEW_SESSION_ODS} as t
|left join (select id,cast(id as string) id_str from pica_ds.pica_doctor a where a.delete_flag = 1 and to_date(a.creat_time) <= '${created_day}') AS b on t.user_id_ods = cast(b.id as string)
|left join (select id,cast(id as string) id_str from pica_ds.pica_doctor a where to_date(a.creat_time) <= '${created_day}') AS b on t.user_id_ods = cast(b.id as string)
|SELECT ss.*,COALESCE(cast(b.id as string),'0') AS user_id from ${MyConfigSession.VIEW_SESSION_NO_MATCH} AS ss
|left join (select distinct id,mobile_phone from pica_ds.pica_doctor where pica_doctor.delete_flag = 1 ) AS b on ss.mobile = b.mobile_phone
|left join (select distinct id,mobile_phone from pica_ds.pica_doctor where delete_flag=1 and mobile_phone!='' and mobile_phone!='XK0HdMN6dAfOlYPOFHHL0A==') AS b on ss.mobile = b.mobile_phone
| select concat(pseudo_session,'0') session_id,min(created_time) session_begin_time,max(created_time) session_end_time from select_res_table
| where pseudo_session not in (select distinct SUBSTRING(refer_session_id,1,length(refer_session_id)-1) from refer_result_table where refer_session_id like '%0')
| group by concat(pseudo_session,'0')
|""".stripMargin
//2.计算所有新拆分的sessionid的起止时间
valsourceSql2=
"""
|select session_id,min(created_time) session_begin_time,max(created_time) session_end_time from refer_result_table
| where SUBSTRING(session_id,1,length(session_id)-1)
| in (select distinct SUBSTRING(refer_session_id,1,length(refer_session_id)-1) from refer_result_table where refer_session_id!='')
| case when t1.menu_end_time is null and t1.action_type='ACTION_VIEW' then t2.session_end_time else t1.menu_end_time end menu_end_time,
| (cast((case when t1.menu_end_time is null and t1.action_type='ACTION_VIEW' then t2.session_end_time else t1.menu_end_time end) as bigint)-cast(t1.menu_begin_time as bigint))/1000 menu_time_diff,
