openreplay/backend/cmd/integrations/main.go
Alexander 4b8f3bee25
Sessions refactoring (#1371)
* feat(backend): moved sql requests related to sessions table to one place

* feat(backend): refactoring in db.Saver handler

* feat(backend): hude refactoring in db/postgres module

* fix(backend): workable feature flags

* fix(backend): workable integrations

* fix(backend): workable sessions and projects modules

* fix(backend): added missed projects module to sessions

* feat(backend): renaming

* feat(backend): moved session struct to sessions module and split methods into interface, cache and storage levels

* feat(backend): moved project struct to projects module

* feat(backend): added projects model

* feat(backend): implemented new in memory cache for sessions and projects

* feat(backend): implemented new cache in projects

* feat(backend): there are 2 methods in cache module now: Get() and GetAndRefresh()

* feat(backend): added cache update operations

* fix(backend): fixed import cycle

* fix(backend): fixed panic in db message handler

* fix(backend): fixed panic in projects module

* fix(backend): fixed panic in sessions.GetDuration

* feat(backend): added direct call to get session duration if session is already in cache

* feat(backend): used pg pool everywhere except db service

* fix(backend): added missing part after rebase

* fix(backend): removed old sessions file

* feat(backend): added refactored redis client with produce/consume options

* feat(backend): added cache layer for projects

* fix(backend): added missing redis config

* fix(backend): added missing method for producer

* feat(backend): cache integration for sessions

* feat(backend): temporary method to get session directly from db

* feat(backend): adapt EE version of message handler

* fix(backend): fixed issue in fts realisation

* fix(backend): added redis cache to sessions module

* fix(backend): set 0 duration or hesitation time for inputs without focus event

* feat(backend): added cache for session updates and failover mechanism for batch.Insert() operation

* feat(backend): debug log

* feat(backend): more debug log

* feat(backend): removed debug log

* fix(backend): fixed an issue of tracking input events with empty label

* fix(backend): disabled debug log in projects cache

* fix(backend): renamed session updater

* fix(backend): fixed closed pool issue in DB service

* fix(backend): fixed dead lock in db Stop() method

* fix(backend): fixed panic in heuristics service

* feat(backend): enabled redis cache in projects

* feat(backend): clear cache on each update operation

* feat(backend): fully integrated cache layer with auto switch

* feat(backend): small refactoring in session updates

* fix(backend): fixed wrong events counter issue

* feat(backend): enabled full cache support in ender and http services

* fix(backend/ee): added missed import

* feat(backend): added second cache layer for db to speed up the service

* feat(backend): disable redis cache

* feat(backend): moved redis cache to ee
2023-07-06 10:55:43 +02:00

114 lines
3.2 KiB
Go

package main
import (
"log"
"openreplay/backend/pkg/db/postgres/pool"
"openreplay/backend/pkg/integrations"
"os"
"os/signal"
"syscall"
"time"
config "openreplay/backend/internal/config/integrations"
"openreplay/backend/internal/integrations/clientManager"
"openreplay/backend/pkg/intervals"
"openreplay/backend/pkg/metrics"
databaseMetrics "openreplay/backend/pkg/metrics/database"
"openreplay/backend/pkg/queue"
"openreplay/backend/pkg/token"
)
func main() {
m := metrics.New()
m.Register(databaseMetrics.List())
log.SetFlags(log.LstdFlags | log.LUTC | log.Llongfile)
cfg := config.New()
// Init postgres connection
pgConn, err := pool.New(cfg.Postgres.String())
if err != nil {
log.Printf("can't init postgres connection: %s", err)
return
}
defer pgConn.Close()
tokenizer := token.NewTokenizer(cfg.TokenSecret)
manager := clientManager.NewManager()
producer := queue.NewProducer(cfg.MessageSizeLimit, true)
defer producer.Close(15000)
// TODO: rework with integration manager
listener, err := integrations.New(pgConn, cfg.Postgres.String())
if err != nil {
log.Printf("Postgres listener error: %v\n", err)
log.Fatalf("Postgres listener error")
}
defer listener.Close()
listener.IterateIntegrationsOrdered(func(i *integrations.Integration, err error) {
if err != nil {
log.Printf("Postgres error: %v\n", err)
return
}
log.Printf("Integration initialization: %v\n", *i)
err = manager.Update(i)
if err != nil {
log.Printf("Integration parse error: %v | Integration: %v\n", err, *i)
return
}
})
sigchan := make(chan os.Signal, 1)
signal.Notify(sigchan, syscall.SIGINT, syscall.SIGTERM)
tick := time.Tick(intervals.INTEGRATIONS_REQUEST_INTERVAL * time.Millisecond)
log.Printf("Integration service started\n")
manager.RequestAll()
for {
select {
case sig := <-sigchan:
log.Printf("Caught signal %v: terminating\n", sig)
listener.Close()
pgConn.Close()
os.Exit(0)
case <-tick:
log.Printf("Requesting all...\n")
manager.RequestAll()
case event := <-manager.Events:
log.Printf("New integration event: %+v\n", *event.IntegrationEvent)
sessionID := event.SessionID
if sessionID == 0 {
sessData, err := tokenizer.Parse(event.Token)
if err != nil && err != token.EXPIRED {
log.Printf("Error on token parsing: %v; Token: %v", err, event.Token)
continue
}
sessionID = sessData.ID
}
producer.Produce(cfg.TopicAnalytics, sessionID, event.IntegrationEvent.Encode())
case err := <-manager.Errors:
log.Printf("Integration error: %v\n", err)
case i := <-manager.RequestDataUpdates:
// log.Printf("Last request integration update: %v || %v\n", i, string(i.RequestData))
if err := listener.UpdateIntegrationRequestData(&i); err != nil {
log.Printf("Postgres Update request_data error: %v\n", err)
}
case err := <-listener.Errors:
log.Printf("Postgres listen error: %v\n", err)
listener.Close()
pgConn.Close()
os.Exit(0)
case iPointer := <-listener.Integrations:
log.Printf("Integration update: %v\n", *iPointer)
err := manager.Update(iPointer)
if err != nil {
log.Printf("Integration parse error: %v | Integration: %v\n", err, *iPointer)
}
}
}
}