openreplay/backend/services/integrations/integration/elasticsearch.go

package integration

import (
	elasticlib "github.com/elastic/go-elasticsearch/v7"
	"context"
	"time"
	"encoding/json"
	"fmt"
	"bytes"
	"strconv"

  "openreplay/backend/pkg/utime"
	"openreplay/backend/pkg/messages"
)


type elasticsearch struct {
 	Host string
 	Port json.Number
 	ApiKeyId string  //`json:"api_key_id"`
 	ApiKey string   //`json:"api_key"`
 	Indexes string
}

type elasticsearchLog struct {
	Message string
	Time time.Time `json:"utc_time"` // Should be parsed automatically from RFC3339
}

type elasticResponce struct {
	Hits struct {
		//Total struct {
		//	Value int
		//}
		Hits []struct {
			Id string `json:"_id"`
			Source json.RawMessage `json:"_source"`
		}
	}
	ScrollId string `json:"_scroll_id"`
}


func (es *elasticsearch) Request(c* client) error {
	address := es.Host + ":" + es.Port.String()
	cfg := elasticlib.Config{
	  Addresses: []string{
	    address,
	  },
		Username: es.ApiKeyId,
		Password: es.ApiKey,
	}
	esC, err := elasticlib.NewClient(cfg)

	if err != nil {
		return err
	}

	// TODO: ping/versions/ client host check
 //  res0, err := esC.Info()
	// if err != nil {
 //     log.Printf("ELASTIC Error getting info: %s", err)
 //  }
 //  defer res0.Body.Close()
 //  // Check response status
 //  if res0.IsError() {
 //    log.Printf("ELASTIC Error: %s", res0.String())
 //  }
 //  log.Printf("ELASTIC  Info: %v ", res0.String())

	gteTs := c.getLastMessageTimestamp() + 1000 // Sec or millisec to add ?

	var buf bytes.Buffer
  query := map[string]interface{}{
    "query": map[string]interface{}{
      "bool": map[string]interface{}{
        "filter": []map[string]interface{}{
        	map[string]interface{}{
						"match": map[string]interface{} {
              "message": map[string]interface{}{
                "query": "openReplaySessionToken=", // asayer_session_id=
              },
            },
        	},
        	map[string]interface{}{
        		"range": map[string]interface{} {
              "utc_time": map[string]interface{}{
                "gte": strconv.FormatUint(gteTs, 10),
                "lte": "now",
              },
            },
        	},
        	map[string]interface{}{
        		"term": map[string]interface{}{
        			"tags": "error",
        		},
        	},
        },
      },
    },
  }
  if err := json.NewEncoder(&buf).Encode(query); err != nil {
    return fmt.Errorf("Error encoding the query: %s", err)
  }

	res, err := esC.Search(
    esC.Search.WithContext(context.Background()),
    esC.Search.WithIndex(es.Indexes),
    esC.Search.WithSize(1000),
    esC.Search.WithScroll(time.Minute * 2),
    esC.Search.WithBody(&buf),
    esC.Search.WithSort("timestamp:asc"),
  )
  if err != nil {
    return fmt.Errorf("Error getting response: %s", err)
  }
  defer res.Body.Close()
  if res.IsError() {
    var e map[string]interface{}
    if err := json.NewDecoder(res.Body).Decode(&e); err != nil {
      return fmt.Errorf("Error parsing the response body: %v", err)
    } else {
      return fmt.Errorf("Elasticsearch [%s] %s: %s",
        res.Status(),
        e["error"],//.(map[string]interface{})["type"],
        e["error"],//.(map[string]interface{})["reason"],
      )
    }
  }

  for {
	  var esResp elasticResponce
	  if err := json.NewDecoder(res.Body).Decode(&esResp); err != nil {
	    return fmt.Errorf("Error parsing the response body: %s", err)
	  }
	  if len(esResp.Hits.Hits) == 0 {
	  	break
	  }

	  for _, hit := range esResp.Hits.Hits {
	  	var esLog elasticsearchLog
			if err = json.Unmarshal(hit.Source, &esLog); err != nil {
				c.errChan <- err
				continue
			}
			token, err := GetToken(esLog.Message)
			if err != nil {
				c.errChan <- err
				continue
			}
			//parsedTime, err := time.Parse(time.RFC3339, esLog.Timestamp)
			//if err != nil {
			//	c.errChan <- err
			//	continue
			//}
			timestamp := uint64(utime.ToMilliseconds(esLog.Time))
			c.setLastMessageTimestamp(timestamp)
			c.evChan <- &SessionErrorEvent{
				//SessionID: sessionID,
				Token: token,
				RawErrorEvent: &messages.RawErrorEvent{
					Source: "elasticsearch",
					Timestamp: timestamp,
					Name: hit.Id, // sure?
					Payload: string(hit.Source),
				},
			}
	  }

	  res, err = esC.Scroll(
	  	esC.Scroll.WithContext(context.Background()),
	  	esC.Scroll.WithScrollID(esResp.ScrollId),
	  	esC.Scroll.WithScroll(time.Minute * 2),
	  )
	  if err != nil {
    	return fmt.Errorf("Error getting scroll response: %s", err)
	  }
	  defer res.Body.Close()
	  if res.IsError() {
	    var e map[string]interface{}
	    if err := json.NewDecoder(res.Body).Decode(&e); err != nil {
	      return fmt.Errorf("Error parsing the response body: %v", err)
	    } else {
	      return fmt.Errorf("Elasticsearch [%s] %s: %s",
	        res.Status(),
	        e["error"],//.(map[string]interface{})["type"],
	        e["error"],//.(map[string]interface{})["reason"],
	      )
	    }
	  }
	}
	return nil
}