28182b951e
* Updated dependancies for redshift connector, changed os module for python-decouple module
* Updated service and images
* Updated message protocol, added exception for BatchMetadata when version is 0 (we apply old read method)
* fixed load error from s3 to redshift. null values for string columns are now empty strings ("")
* Added file test consumer_async.py: reads every 3 minutes kafka raw and send task in background to upload to cloud
* Added method to skip messages that are not inserted to cloud
* Added logs into consumer_async. Changed urls and issues in sessions table from list to string
* Split between messages for sessions table and for events table
* Updated redshift tables
* Fixed small issue in query redshift_sessions.sql
* Updated Dockerfiles. Cleaned logs of consumer_async. Updated/Fixed tables. Transformed Nan as NULL for VARCHAR columns
* Added error handler for sql dropped connection
* chore(docker): Optimize docker builds
Signed-off-by: rjshrjndrn <rjshrjndrn@gmail.com>
* Variables renamed
* Adding compression libraries
* Set default value of count events to 0 (instead of NULL) when event did not occur
* Added support specific project tracking. Added PG handler to connect to sessions table
* Added method to update values in db connection for sessions ended and restarted
* Removing intelligent file copying
* chore(connector): Build file
Signed-off-by: rjshrjndrn <rjshrjndrn@gmail.com>
* Adding connection pool for pg
* Renaming and optimizing
* Fixed issue of missing information of sessions
---------
Signed-off-by: rjshrjndrn <rjshrjndrn@gmail.com>
Co-authored-by: rjshrjndrn <rjshrjndrn@gmail.com>
91 lines
3.1 KiB
SQL
91 lines
3.1 KiB
SQL
CREATE TABLE IF NOT EXISTS connector_events_detailed
|
|
(
|
|
sessionid BIGINT,
|
|
clickevent_hesitationtime BIGINT,
|
|
clickevent_label VARCHAR(5000),
|
|
clickevent_messageid BIGINT,
|
|
clickevent_timestamp BIGINT,
|
|
connectioninformation_downlink BIGINT,
|
|
connectioninformation_type VARCHAR(5000),
|
|
consolelog_level VARCHAR(5000),
|
|
consolelog_value VARCHAR(5000),
|
|
customevent_name VARCHAR(5000),
|
|
customevent_payload VARCHAR(5000),
|
|
fetch_duration BIGINT,
|
|
fetch_method VARCHAR(5000),
|
|
fetch_request VARCHAR(5000),
|
|
fetch_response VARCHAR(5000),
|
|
fetch_status BIGINT,
|
|
fetch_timestamp BIGINT,
|
|
fetch_url VARCHAR(5000),
|
|
graphql_operationkind VARCHAR(5000),
|
|
graphql_operationname VARCHAR(5000),
|
|
graphql_response VARCHAR(5000),
|
|
graphql_variables VARCHAR(5000),
|
|
inputevent_label VARCHAR(5000),
|
|
inputevent_messageid BIGINT,
|
|
inputevent_timestamp BIGINT,
|
|
inputevent_value VARCHAR(5000),
|
|
inputevent_valuemasked BOOLEAN,
|
|
jsexception_message VARCHAR(5000),
|
|
jsexception_name VARCHAR(5000),
|
|
jsexception_payload VARCHAR(5000),
|
|
jsexception_metadata VARCHAR(5000),
|
|
mouseclick_id BIGINT,
|
|
mouseclick_hesitationtime BIGINT,
|
|
mouseclick_label VARCHAR(5000),
|
|
networkrequest_type VARCHAR(5000),
|
|
networkrequest_method VARCHAR(5000),
|
|
networkrequest_url VARCHAR(5000),
|
|
networkrequest_request VARCHAR(5000),
|
|
networkrequest_response VARCHAR(5000),
|
|
networkrequest_status BIGINT,
|
|
networkrequest_timestamp BIGINT,
|
|
networkrequest_duration BIGINT,
|
|
pageevent_domcontentloadedeventend BIGINT,
|
|
pageevent_domcontentloadedeventstart BIGINT,
|
|
pageevent_firstcontentfulpaint BIGINT,
|
|
pageevent_firstpaint BIGINT,
|
|
pageevent_loaded BOOLEAN,
|
|
pageevent_loadeventend BIGINT,
|
|
pageevent_loadeventstart BIGINT,
|
|
pageevent_messageid BIGINT,
|
|
pageevent_referrer VARCHAR(5000),
|
|
pageevent_requeststart BIGINT,
|
|
pageevent_responseend BIGINT,
|
|
pageevent_responsestart BIGINT,
|
|
pageevent_speedindex BIGINT,
|
|
pageevent_timestamp BIGINT,
|
|
pageevent_url VARCHAR(5000),
|
|
sessionend_timestamp BIGINT,
|
|
sessionend_encryption_key VARCHAR(5000),
|
|
sessionstart_projectid BIGINT,
|
|
sessionstart_revid VARCHAR(5000),
|
|
sessionstart_timestamp BIGINT,
|
|
sessionstart_trackerversion VARCHAR(5000),
|
|
sessionstart_useragent VARCHAR(5000),
|
|
sessionstart_userbrowser VARCHAR(5000),
|
|
sessionstart_userbrowserversion VARCHAR(5000),
|
|
sessionstart_usercountry VARCHAR(5000),
|
|
sessionstart_userdevice VARCHAR(5000),
|
|
sessionstart_userdeviceheapsize BIGINT,
|
|
sessionstart_userdevicememorysize BIGINT,
|
|
sessionstart_userdevicetype VARCHAR(5000),
|
|
sessionstart_useros VARCHAR(5000),
|
|
sessionstart_userosversion VARCHAR(5000),
|
|
sessionstart_useruuid VARCHAR(5000),
|
|
setpagelocation_navigationstart BIGINT,
|
|
setpagelocation_referrer VARCHAR(5000),
|
|
setpagelocation_url VARCHAR(5000),
|
|
issueevent_message_id BIGINT,
|
|
issueevent_timestamp BIGINT,
|
|
issueevent_type VARCHAR(5000),
|
|
issueevent_context_string VARCHAR(5000),
|
|
issueevent_context VARCHAR(5000),
|
|
issueevent_payload VARCHAR(5000),
|
|
issueevent_url VARCHAR(5000),
|
|
customissue_name VARCHAR(5000),
|
|
customissue_payload VARCHAR(5000),
|
|
received_at BIGINT,
|
|
batch_order_number BIGINT
|
|
);
|