diff --git a/backend/Dockerfile b/backend/Dockerfile index 5cefd4cb4..4c31518ef 100644 --- a/backend/Dockerfile +++ b/backend/Dockerfile @@ -1,4 +1,4 @@ -FROM golang:1.13-alpine3.10 AS prepare +FROM golang:1.18-alpine3.15 AS prepare RUN apk add --no-cache git openssh openssl-dev pkgconf gcc g++ make libc-dev bash diff --git a/backend/go.mod b/backend/go.mod index ab98ca444..6588529a8 100644 --- a/backend/go.mod +++ b/backend/go.mod @@ -1,14 +1,12 @@ module openreplay/backend -go 1.13 +go 1.18 require ( cloud.google.com/go/logging v1.4.2 github.com/ClickHouse/clickhouse-go v1.4.3 - github.com/Masterminds/squirrel v1.5.0 github.com/aws/aws-sdk-go v1.35.23 github.com/btcsuite/btcutil v1.0.2 - github.com/confluentinc/confluent-kafka-go v1.7.0 // indirect github.com/elastic/go-elasticsearch/v7 v7.13.1 github.com/go-redis/redis v6.15.9+incompatible github.com/google/uuid v1.1.2 @@ -16,14 +14,47 @@ require ( github.com/jackc/pgconn v1.6.0 github.com/jackc/pgerrcode v0.0.0-20201024163028-a0d42d470451 github.com/jackc/pgx/v4 v4.6.0 - github.com/klauspost/compress v1.11.9 // indirect github.com/klauspost/pgzip v1.2.5 - github.com/lib/pq v1.2.0 github.com/oschwald/maxminddb-golang v1.7.0 github.com/pkg/errors v0.9.1 github.com/tomasen/realip v0.0.0-20180522021738-f0c99a92ddce github.com/ua-parser/uap-go v0.0.0-20200325213135-e1c09f13e2fe + golang.org/x/net v0.0.0-20210503060351-7fd8e65b6420 google.golang.org/api v0.50.0 gopkg.in/confluentinc/confluent-kafka-go.v1 v1.7.0 - +) + +require ( + cloud.google.com/go v0.84.0 // indirect + github.com/cloudflare/golz4 v0.0.0-20150217214814-ef862a3cdc58 // indirect + github.com/confluentinc/confluent-kafka-go v1.7.0 // indirect + github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e // indirect + github.com/golang/protobuf v1.5.2 // indirect + github.com/google/go-cmp v0.5.6 // indirect + github.com/googleapis/gax-go/v2 v2.0.5 // indirect + github.com/jackc/chunkreader/v2 v2.0.1 // indirect + github.com/jackc/pgio v1.0.0 // indirect + github.com/jackc/pgpassfile v1.0.0 // indirect + github.com/jackc/pgproto3/v2 v2.0.2 // indirect + github.com/jackc/pgservicefile v0.0.0-20200307190119-3430c5407db8 // indirect + github.com/jackc/pgtype v1.3.0 // indirect + github.com/jackc/puddle v1.1.0 // indirect + github.com/jmespath/go-jmespath v0.4.0 // indirect + github.com/jstemmer/go-junit-report v0.9.1 // indirect + github.com/klauspost/compress v1.11.9 // indirect + go.opencensus.io v0.23.0 // indirect + golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9 // indirect + golang.org/x/lint v0.0.0-20210508222113-6edffad5e616 // indirect + golang.org/x/mod v0.4.2 // indirect + golang.org/x/oauth2 v0.0.0-20210628180205-a41e5a781914 // indirect + golang.org/x/sync v0.0.0-20210220032951-036812b2e83c // indirect + golang.org/x/sys v0.0.0-20210616094352-59db8d763f22 // indirect + golang.org/x/text v0.3.6 // indirect + golang.org/x/tools v0.1.4 // indirect + golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 // indirect + google.golang.org/appengine v1.6.7 // indirect + google.golang.org/genproto v0.0.0-20210624195500-8bfb893ecb84 // indirect + google.golang.org/grpc v1.38.0 // indirect + google.golang.org/protobuf v1.26.0 // indirect + gopkg.in/yaml.v2 v2.2.8 // indirect ) diff --git a/backend/go.sum b/backend/go.sum index 8d538a0b4..607936204 100644 --- a/backend/go.sum +++ b/backend/go.sum @@ -46,8 +46,6 @@ github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03 github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= github.com/ClickHouse/clickhouse-go v1.4.3 h1:iAFMa2UrQdR5bHJ2/yaSLffZkxpcOYQMCUuKeNXGdqc= github.com/ClickHouse/clickhouse-go v1.4.3/go.mod h1:EaI/sW7Azgz9UATzd5ZdZHRUhHgv5+JMS9NSr2smCJI= -github.com/Masterminds/squirrel v1.5.0 h1:JukIZisrUXadA9pl3rMkjhiamxiB0cXiu+HGp/Y8cY8= -github.com/Masterminds/squirrel v1.5.0/go.mod h1:NNaOrjSoIDfDA40n7sr2tPNZRfjzjA400rg+riTZj10= github.com/aead/siphash v1.0.1/go.mod h1:Nywa3cDsYNNK3gaciGTWPwHt0wlpNV15vwmswBAUSII= github.com/aws/aws-sdk-go v1.35.23 h1:SCP0d0XvyJTDmfnHEQPvBaYi3kea1VNUo7uQmkVgFts= github.com/aws/aws-sdk-go v1.35.23/go.mod h1:tlPOdRjfxPBpNIwqDj61rmsnA85v9jc0Ps9+muhnW+k= @@ -75,8 +73,8 @@ github.com/cncf/udpa/go v0.0.0-20200629203442-efcf912fb354/go.mod h1:WmhPx2Nbnht github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= github.com/cockroachdb/apd v1.1.0 h1:3LFP3629v+1aKXU5Q37mxmRxX/pIu1nijXydLShEq5I= github.com/cockroachdb/apd v1.1.0/go.mod h1:8Sl8LxpKi29FqWXR16WEFZRNSz3SoPzUzeMeY4+DwBQ= -github.com/confluentinc/confluent-kafka-go v1.5.2 h1:l+qt+a0Okmq0Bdr1P55IX4fiwFJyg0lZQmfHkAFkv7E= -github.com/confluentinc/confluent-kafka-go v1.5.2/go.mod h1:u2zNLny2xq+5rWeTQjFHbDzzNuba4P1vo31r9r4uAdg= +github.com/confluentinc/confluent-kafka-go v1.7.0 h1:tXh3LWb2Ne0WiU3ng4h5qiGA9XV61rz46w60O+cq8bM= +github.com/confluentinc/confluent-kafka-go v1.7.0/go.mod h1:u2zNLny2xq+5rWeTQjFHbDzzNuba4P1vo31r9r4uAdg= github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= github.com/coreos/go-systemd v0.0.0-20190719114852-fd7a80b32e1f/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7DoTY= @@ -93,7 +91,6 @@ github.com/envoyproxy/go-control-plane v0.9.7/go.mod h1:cwu0lG7PUMfa9snN8LXBig5y github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= github.com/envoyproxy/go-control-plane v0.9.9-0.20210217033140-668b12f5399d/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= -github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= @@ -135,7 +132,6 @@ github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaS github.com/golang/protobuf v1.5.1/go.mod h1:DopwsBzvsk0Fs44TXzsVbJyPhcCPeIwnvohx4u74HPM= github.com/golang/protobuf v1.5.2 h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw= github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= -github.com/golang/snappy v0.0.3 h1:fHPg5GQYlCeLIPB9BZqMVR5nR9A+IM5zcgeTdjMYmLA= github.com/golang/snappy v0.0.3/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= @@ -152,11 +148,9 @@ github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/ github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.6 h1:BKbKCqvP6I+rmFHt06ZmyQtvB8xAkWdhFyr0ZUNZcxQ= github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/martian v2.1.0+incompatible h1:/CP5g8u/VJHijgedC/Legn3BAbAaWPgecwXBIDzw5no= github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= github.com/google/martian/v3 v3.1.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= -github.com/google/martian/v3 v3.2.1 h1:d8MncMlErDFTwQGBK1xhv026j9kqhvw1Qv9IbWT1VLQ= github.com/google/martian/v3 v3.2.1/go.mod h1:oBOf6HBosgwRXnUGWUB05QECsc6uvmMiJ3+6W4l/CUk= github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= @@ -184,7 +178,6 @@ github.com/hpcloud/tail v1.0.0 h1:nfCOvKYfkgYP8hkirhJocXT2+zOD8yUNjXaWfTlyFKI= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= -github.com/jackc/chunkreader v1.0.0 h1:4s39bBR8ByfqH+DKm8rQA3E1LHZWB9XWcrz8fqaZbe0= github.com/jackc/chunkreader v1.0.0/go.mod h1:RT6O25fNZIuasFJRyZ4R/Y2BbhasbmZXF9QQ7T3kePo= github.com/jackc/chunkreader/v2 v2.0.0/go.mod h1:odVSm741yZoC3dpHEUXIqA9tQRhFrgOHwnPIn9lDKlk= github.com/jackc/chunkreader/v2 v2.0.1 h1:i+RDz65UE+mmpjTfyz0MoVTnzeYxroil2G82ki7MGG8= @@ -203,7 +196,6 @@ github.com/jackc/pgmock v0.0.0-20190831213851-13a1b77aafa2 h1:JVX6jT/XfzNqIjye47 github.com/jackc/pgmock v0.0.0-20190831213851-13a1b77aafa2/go.mod h1:fGZlG77KXmcq05nJLRkk0+p82V8B8Dw8KN2/V9c/OAE= github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM= github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg= -github.com/jackc/pgproto3 v1.1.0 h1:FYYE4yRw+AgI8wXIinMlNjBbp/UitDJwfj5LqqewP1A= github.com/jackc/pgproto3 v1.1.0/go.mod h1:eR5FA3leWg7p9aeAqi37XOTgTIbkABlvcPB3E5rlc78= github.com/jackc/pgproto3/v2 v2.0.0-alpha1.0.20190420180111-c116219b62db/go.mod h1:bhq50y+xrl9n5mRYyCBFKkpRVTLYJVWeCc+mEAI3yXA= github.com/jackc/pgproto3/v2 v2.0.0-alpha1.0.20190609003834-432c2951c711/go.mod h1:uH0AWtUmuShn0bcesswc4aBTWGvw0cAxIJp+6OB//Wg= @@ -219,7 +211,6 @@ github.com/jackc/pgtype v0.0.0-20190824184912-ab885b375b90/go.mod h1:KcahbBH1nCM github.com/jackc/pgtype v0.0.0-20190828014616-a8802b16cc59/go.mod h1:MWlu30kVJrUS8lot6TQqcg7mtthZ9T0EoIBFiJcmcyw= github.com/jackc/pgtype v1.3.0 h1:l8JvKrby3RI7Kg3bYEeU9TA4vqC38QDpFCfcrC7KuN0= github.com/jackc/pgtype v1.3.0/go.mod h1:b0JqxHvPmljG+HQ5IsvQ0yqeSi4nGcDTVjFoiLDb0Ik= -github.com/jackc/pgx v3.6.2+incompatible h1:2zP5OD7kiyR3xzRYMhOcXVvkDZsImVXfj+yIyTQf3/o= github.com/jackc/pgx v3.6.2+incompatible/go.mod h1:0ZGrqGqkRlliWnWB4zKnWtjbSWbGkVEFm4TeybAXq+I= github.com/jackc/pgx/v4 v4.0.0-20190420224344-cc3461e65d96/go.mod h1:mdxmSJJuR08CZQyj1PVQBHy9XOp5p8/SHH6a0psbY9Y= github.com/jackc/pgx/v4 v4.0.0-20190421002000-1b8f0016e912/go.mod h1:no/Y67Jkk/9WuGR0JG/JseM9irFbnEPbuWV2EELPNuM= @@ -254,10 +245,6 @@ github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/pty v1.1.8/go.mod h1:O1sed60cT9XZ5uDucP5qwvh+TE3NnUj51EiZO/lmSfw= github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= -github.com/lann/builder v0.0.0-20180802200727-47ae307949d0 h1:SOEGU9fKiNWd/HOJuq6+3iTQz8KNCLtVX6idSoTLdUw= -github.com/lann/builder v0.0.0-20180802200727-47ae307949d0/go.mod h1:dXGbAdH5GtBTC4WfIxhKZfyBF/HBFgRZSWwZ9g/He9o= -github.com/lann/ps v0.0.0-20150810152359-62de8c46ede0 h1:P6pPBnrTSX3DEVR4fDembhRWSsG5rVo6hYhAB/ADZrk= -github.com/lann/ps v0.0.0-20150810152359-62de8c46ede0/go.mod h1:vmVJ0l/dxyfGW6FmdpVm2joNMFikkuWg0EoCKLGUMNw= github.com/lib/pq v1.0.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= github.com/lib/pq v1.1.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= github.com/lib/pq v1.2.0 h1:LXpIM/LZ5xGFhOpXAQUIMM1HdyqzVYM13zNdjCEEcA0= @@ -682,8 +669,8 @@ google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQ gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/confluentinc/confluent-kafka-go.v1 v1.5.2 h1:g0WBLy6fobNUU8W/e9zx6I0Yl79Ya+BDW1NwzAlTiiQ= -gopkg.in/confluentinc/confluent-kafka-go.v1 v1.5.2/go.mod h1:ZdI3yfYmdNSLQPNCpO1y00EHyWaHG5EnQEyL/ntAegY= +gopkg.in/confluentinc/confluent-kafka-go.v1 v1.7.0 h1:+RlmciBLDd/XwM1iudiG3HtCg45purnsOxEoY/+JZdQ= +gopkg.in/confluentinc/confluent-kafka-go.v1 v1.7.0/go.mod h1:ZdI3yfYmdNSLQPNCpO1y00EHyWaHG5EnQEyL/ntAegY= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/fsnotify.v1 v1.4.7 h1:xOHLXZwVvI9hhs+cLKq5+I5onOuwQLhQwiu63xxlHs4= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= diff --git a/backend/pkg/db/cache/messages_common.go b/backend/pkg/db/cache/messages-common.go similarity index 66% rename from backend/pkg/db/cache/messages_common.go rename to backend/pkg/db/cache/messages-common.go index 3983982fe..8ca7b2f85 100644 --- a/backend/pkg/db/cache/messages_common.go +++ b/backend/pkg/db/cache/messages-common.go @@ -28,30 +28,6 @@ func (c *PGCache) InsertIssueEvent(sessionID uint64, crash *IssueEvent) error { return c.Conn.InsertIssueEvent(sessionID, session.ProjectID, crash) } -func (c *PGCache) InsertUserID(sessionID uint64, userID *IOSUserID) error { - if err := c.Conn.InsertIOSUserID(sessionID, userID); err != nil { - return err - } - session, err := c.GetSession(sessionID) - if err != nil { - return err - } - session.UserID = &userID.Value - return nil -} - -func (c *PGCache) InsertUserAnonymousID(sessionID uint64, userAnonymousID *IOSUserAnonymousID) error { - if err := c.Conn.InsertIOSUserAnonymousID(sessionID, userAnonymousID); err != nil { - return err - } - session, err := c.GetSession(sessionID) - if err != nil { - return err - } - session.UserAnonymousID = &userAnonymousID.Value - return nil -} - func (c *PGCache) InsertMetadata(sessionID uint64, metadata *Metadata) error { session, err := c.GetSession(sessionID) if err != nil { diff --git a/backend/pkg/db/cache/messages_ios.go b/backend/pkg/db/cache/messages-ios.go similarity index 100% rename from backend/pkg/db/cache/messages_ios.go rename to backend/pkg/db/cache/messages-ios.go diff --git a/backend/pkg/db/cache/messages_web.go b/backend/pkg/db/cache/messages-web.go similarity index 100% rename from backend/pkg/db/cache/messages_web.go rename to backend/pkg/db/cache/messages-web.go diff --git a/backend/pkg/db/cache/pg_cache.go b/backend/pkg/db/cache/pg-cache.go similarity index 100% rename from backend/pkg/db/cache/pg_cache.go rename to backend/pkg/db/cache/pg-cache.go diff --git a/backend/pkg/db/postgres/listener.go b/backend/pkg/db/postgres/listener.go index f90d83485..ef99c2c59 100644 --- a/backend/pkg/db/postgres/listener.go +++ b/backend/pkg/db/postgres/listener.go @@ -11,7 +11,6 @@ import ( type Listener struct { conn *pgx.Conn Integrations chan *Integration - Alerts chan *Alert Errors chan error } @@ -32,23 +31,6 @@ func NewIntegrationsListener(url string) (*Listener, error) { return listener, nil } -func NewAlertsListener(url string) (*Listener, error) { - conn, err := pgx.Connect(context.Background(), url) - if err != nil { - return nil, err - } - listener := &Listener{ - conn: conn, - Errors: make(chan error), - } - listener.Alerts = make(chan *Alert, 50) - if _, err := conn.Exec(context.Background(), "LISTEN alert"); err != nil { - return nil, err - } - go listener.listen() - return listener, nil -} - func (listener *Listener) listen() { for { notification, err := listener.conn.WaitForNotification(context.Background()) @@ -64,13 +46,6 @@ func (listener *Listener) listen() { } else { listener.Integrations <- integrationP } - case "alert": - alertP := new(Alert) - if err := json.Unmarshal([]byte(notification.Payload), alertP); err != nil { - listener.Errors <- fmt.Errorf("%v | Payload: %v", err, notification.Payload) - } else { - listener.Alerts <- alertP - } } } } diff --git a/backend/pkg/db/postgres/messages_common.go b/backend/pkg/db/postgres/messages-common.go similarity index 100% rename from backend/pkg/db/postgres/messages_common.go rename to backend/pkg/db/postgres/messages-common.go diff --git a/backend/pkg/db/postgres/messages_ios.go b/backend/pkg/db/postgres/messages-ios.go similarity index 100% rename from backend/pkg/db/postgres/messages_ios.go rename to backend/pkg/db/postgres/messages-ios.go diff --git a/backend/pkg/db/postgres/messages_web_stats.go b/backend/pkg/db/postgres/messages-web-stats.go similarity index 100% rename from backend/pkg/db/postgres/messages_web_stats.go rename to backend/pkg/db/postgres/messages-web-stats.go diff --git a/backend/pkg/db/postgres/messages_web.go b/backend/pkg/db/postgres/messages-web.go similarity index 100% rename from backend/pkg/db/postgres/messages_web.go rename to backend/pkg/db/postgres/messages-web.go diff --git a/backend/pkg/db/postgres/unstarted_session.go b/backend/pkg/db/postgres/unstarted-session.go similarity index 100% rename from backend/pkg/db/postgres/unstarted_session.go rename to backend/pkg/db/postgres/unstarted-session.go diff --git a/backend/pkg/env/worker_id.go b/backend/pkg/env/worker-id.go similarity index 100% rename from backend/pkg/env/worker_id.go rename to backend/pkg/env/worker-id.go diff --git a/backend/pkg/messages/get_timestamp.go b/backend/pkg/messages/get-timestamp.go similarity index 100% rename from backend/pkg/messages/get_timestamp.go rename to backend/pkg/messages/get-timestamp.go diff --git a/backend/pkg/messages/legacy_message_transform.go b/backend/pkg/messages/legacy-message-transform.go similarity index 100% rename from backend/pkg/messages/legacy_message_transform.go rename to backend/pkg/messages/legacy-message-transform.go diff --git a/backend/pkg/messages/read_message.go b/backend/pkg/messages/read-message.go similarity index 100% rename from backend/pkg/messages/read_message.go rename to backend/pkg/messages/read-message.go diff --git a/backend/services/http/handlers_depricated.go b/backend/services/http/handlers-depricated.go similarity index 100% rename from backend/services/http/handlers_depricated.go rename to backend/services/http/handlers-depricated.go diff --git a/backend/services/http/handlers_ios.go b/backend/services/http/handlers-ios.go similarity index 71% rename from backend/services/http/handlers_ios.go rename to backend/services/http/handlers-ios.go index 6c3f945bd..affcab59d 100644 --- a/backend/services/http/handlers_ios.go +++ b/backend/services/http/handlers-ios.go @@ -2,50 +2,50 @@ package main import ( "encoding/json" - "net/http" "errors" - "time" - "math/rand" - "strconv" "log" + "math/rand" + "net/http" + "strconv" + "time" "openreplay/backend/pkg/db/postgres" - "openreplay/backend/pkg/token" . "openreplay/backend/pkg/messages" + "openreplay/backend/pkg/token" ) -const FILES_SIZE_LIMIT int64 = 1e7 // 10Mb +const FILES_SIZE_LIMIT int64 = 1e7 // 10Mb func startSessionHandlerIOS(w http.ResponseWriter, r *http.Request) { type request struct { - Token string `json:"token"` - ProjectKey *string `json:"projectKey"` - TrackerVersion string `json:"trackerVersion"` - RevID string `json:"revID"` - UserUUID *string `json:"userUUID"` + Token string `json:"token"` + ProjectKey *string `json:"projectKey"` + TrackerVersion string `json:"trackerVersion"` + RevID string `json:"revID"` + UserUUID *string `json:"userUUID"` //UserOS string `json"userOS"` //hardcoded 'MacOS' - UserOSVersion string `json:"userOSVersion"` - UserDevice string `json:"userDevice"` - Timestamp uint64 `json:"timestamp"` + UserOSVersion string `json:"userOSVersion"` + UserDevice string `json:"userDevice"` + Timestamp uint64 `json:"timestamp"` // UserDeviceType uint 0:phone 1:pad 2:tv 3:carPlay 5:mac // “performances”:{ - // “activeProcessorCount”:8, - // “isLowPowerModeEnabled”:0, - // “orientation”:0, - // “systemUptime”:585430, - // “batteryState”:0, - // “thermalState”:0, - // “batteryLevel”:0, - // “processorCount”:8, - // “physicalMemory”:17179869184 - // }, + // “activeProcessorCount”:8, + // “isLowPowerModeEnabled”:0, + // “orientation”:0, + // “systemUptime”:585430, + // “batteryState”:0, + // “thermalState”:0, + // “batteryLevel”:0, + // “processorCount”:8, + // “physicalMemory”:17179869184 + // }, } type response struct { - Token string `json:"token"` - ImagesHashList []string `json:"imagesHashList"` - UserUUID string `json:"userUUID"` - BeaconSizeLimit int64 `json:"beaconSizeLimit"` - SessionID string `json:"sessionID"` + Token string `json:"token"` + ImagesHashList []string `json:"imagesHashList"` + UserUUID string `json:"userUUID"` + BeaconSizeLimit int64 `json:"beaconSizeLimit"` + SessionID string `json:"sessionID"` } startTime := time.Now() req := &request{} @@ -98,16 +98,16 @@ func startSessionHandlerIOS(w http.ResponseWriter, r *http.Request) { // The difference with web is mostly here: producer.Produce(TOPIC_RAW_IOS, tokenData.ID, Encode(&IOSSessionStart{ - Timestamp: req.Timestamp, - ProjectID: uint64(p.ProjectID), - TrackerVersion: req.TrackerVersion, - RevID: req.RevID, - UserUUID: userUUID, - UserOS: "IOS", - UserOSVersion: req.UserOSVersion, - UserDevice: MapIOSDevice(req.UserDevice), - UserDeviceType: GetIOSDeviceType(req.UserDevice), - UserCountry: country, + Timestamp: req.Timestamp, + ProjectID: uint64(p.ProjectID), + TrackerVersion: req.TrackerVersion, + RevID: req.RevID, + UserUUID: userUUID, + UserOS: "IOS", + UserOSVersion: req.UserOSVersion, + UserDevice: MapIOSDevice(req.UserDevice), + UserDeviceType: GetIOSDeviceType(req.UserDevice), + UserCountry: country, })) } @@ -119,14 +119,13 @@ func startSessionHandlerIOS(w http.ResponseWriter, r *http.Request) { responseWithJSON(w, &response{ // ImagesHashList: imagesHashList, - Token: tokenizer.Compose(*tokenData), - UserUUID: userUUID, - SessionID: strconv.FormatUint(tokenData.ID, 10), + Token: tokenizer.Compose(*tokenData), + UserUUID: userUUID, + SessionID: strconv.FormatUint(tokenData.ID, 10), BeaconSizeLimit: BEACON_SIZE_LIMIT, }) } - func pushMessagesHandlerIOS(w http.ResponseWriter, r *http.Request) { sessionData, err := tokenizer.ParseFromHTTPRequest(r) if err != nil { @@ -136,8 +135,6 @@ func pushMessagesHandlerIOS(w http.ResponseWriter, r *http.Request) { pushMessages(w, r, sessionData.ID, TOPIC_RAW_IOS) } - - func pushLateMessagesHandlerIOS(w http.ResponseWriter, r *http.Request) { sessionData, err := tokenizer.ParseFromHTTPRequest(r) if err != nil && err != token.EXPIRED { @@ -145,10 +142,9 @@ func pushLateMessagesHandlerIOS(w http.ResponseWriter, r *http.Request) { return } // Check timestamps here? - pushMessages(w, r, sessionData.ID,TOPIC_RAW_IOS) + pushMessages(w, r, sessionData.ID, TOPIC_RAW_IOS) } - func imagesUploadHandlerIOS(w http.ResponseWriter, r *http.Request) { log.Printf("recieved imagerequest") @@ -163,12 +159,12 @@ func imagesUploadHandlerIOS(w http.ResponseWriter, r *http.Request) { err = r.ParseMultipartForm(1e6) // ~1Mb if err == http.ErrNotMultipart || err == http.ErrMissingBoundary { responseWithError(w, http.StatusUnsupportedMediaType, err) - // } else if err == multipart.ErrMessageTooLarge // if non-files part exceeds 10 MB + // } else if err == multipart.ErrMessageTooLarge // if non-files part exceeds 10 MB } else if err != nil { responseWithError(w, http.StatusInternalServerError, err) // TODO: send error here only on staging } - if (r.MultipartForm == nil) { + if r.MultipartForm == nil { responseWithError(w, http.StatusInternalServerError, errors.New("Multipart not parsed")) } @@ -177,7 +173,7 @@ func imagesUploadHandlerIOS(w http.ResponseWriter, r *http.Request) { return } - prefix := r.MultipartForm.Value["projectKey"][0] + "/" + strconv.FormatUint(sessionData.ID, 10) + "/" + prefix := r.MultipartForm.Value["projectKey"][0] + "/" + strconv.FormatUint(sessionData.ID, 10) + "/" for _, fileHeaderList := range r.MultipartForm.File { for _, fileHeader := range fileHeaderList { @@ -187,7 +183,7 @@ func imagesUploadHandlerIOS(w http.ResponseWriter, r *http.Request) { } key := prefix + fileHeader.Filename log.Printf("Uploading image... %v", key) - go func() { //TODO: mime type from header + go func() { //TODO: mime type from header if err := s3.Upload(file, key, "image/jpeg", false); err != nil { log.Printf("Upload ios screen error. %v", err) } diff --git a/backend/services/http/handlers_web.go b/backend/services/http/handlers-web.go similarity index 100% rename from backend/services/http/handlers_web.go rename to backend/services/http/handlers-web.go diff --git a/backend/services/http/ios-device.go b/backend/services/http/ios-device.go new file mode 100644 index 000000000..bec1f3b36 --- /dev/null +++ b/backend/services/http/ios-device.go @@ -0,0 +1,138 @@ +package main + +import ( + "strings" +) + +func MapIOSDevice(identifier string) string { + switch identifier { + case "iPod5,1": + return "iPod touch (5th generation)" + case "iPod7,1": + return "iPod touch (6th generation)" + case "iPod9,1": + return "iPod touch (7th generation)" + case "iPhone3,1", "iPhone3,2", "iPhone3,3": + return "iPhone 4" + case "iPhone4,1": + return "iPhone 4s" + case "iPhone5,1", "iPhone5,2": + return "iPhone 5" + case "iPhone5,3", "iPhone5,4": + return "iPhone 5c" + case "iPhone6,1", "iPhone6,2": + return "iPhone 5s" + case "iPhone7,2": + return "iPhone 6" + case "iPhone7,1": + return "iPhone 6 Plus" + case "iPhone8,1": + return "iPhone 6s" + case "iPhone8,2": + return "iPhone 6s Plus" + case "iPhone8,4": + return "iPhone SE" + case "iPhone9,1", "iPhone9,3": + return "iPhone 7" + case "iPhone9,2", "iPhone9,4": + return "iPhone 7 Plus" + case "iPhone10,1", "iPhone10,4": + return "iPhone 8" + case "iPhone10,2", "iPhone10,5": + return "iPhone 8 Plus" + case "iPhone10,3", "iPhone10,6": + return "iPhone X" + case "iPhone11,2": + return "iPhone XS" + case "iPhone11,4", "iPhone11,6": + return "iPhone XS Max" + case "iPhone11,8": + return "iPhone XR" + case "iPhone12,1": + return "iPhone 11" + case "iPhone12,3": + return "iPhone 11 Pro" + case "iPhone12,5": + return "iPhone 11 Pro Max" + case "iPhone12,8": + return "iPhone SE (2nd generation)" + case "iPhone13,1": + return "iPhone 12 mini" + case "iPhone13,2": + return "iPhone 12" + case "iPhone13,3": + return "iPhone 12 Pro" + case "iPhone13,4": + return "iPhone 12 Pro Max" + case "iPad2,1", "iPad2,2", "iPad2,3", "iPad2,4": + return "iPad 2" + case "iPad3,1", "iPad3,2", "iPad3,3": + return "iPad (3rd generation)" + case "iPad3,4", "iPad3,5", "iPad3,6": + return "iPad (4th generation)" + case "iPad6,11", "iPad6,12": + return "iPad (5th generation)" + case "iPad7,5", "iPad7,6": + return "iPad (6th generation)" + case "iPad7,11", "iPad7,12": + return "iPad (7th generation)" + case "iPad11,6", "iPad11,7": + return "iPad (8th generation)" + case "iPad4,1", "iPad4,2", "iPad4,3": + return "iPad Air" + case "iPad5,3", "iPad5,4": + return "iPad Air 2" + case "iPad11,3", "iPad11,4": + return "iPad Air (3rd generation)" + case "iPad13,1", "iPad13,2": + return "iPad Air (4th generation)" + case "iPad2,5", "iPad2,6", "iPad2,7": + return "iPad mini" + case "iPad4,4", "iPad4,5", "iPad4,6": + return "iPad mini 2" + case "iPad4,7", "iPad4,8", "iPad4,9": + return "iPad mini 3" + case "iPad5,1", "iPad5,2": + return "iPad mini 4" + case "iPad11,1", "iPad11,2": + return "iPad mini (5th generation)" + case "iPad6,3", "iPad6,4": + return "iPad Pro (9.7-inch)" + case "iPad7,3", "iPad7,4": + return "iPad Pro (10.5-inch)" + case "iPad8,1", "iPad8,2", "iPad8,3", "iPad8,4": + return "iPad Pro (11-inch) (1st generation)" + case "iPad8,9", "iPad8,10": + return "iPad Pro (11-inch) (2nd generation)" + case "iPad6,7", "iPad6,8": + return "iPad Pro (12.9-inch) (1st generation)" + case "iPad7,1", "iPad7,2": + return "iPad Pro (12.9-inch) (2nd generation)" + case "iPad8,5", "iPad8,6", "iPad8,7", "iPad8,8": + return "iPad Pro (12.9-inch) (3rd generation)" + case "iPad8,11", "iPad8,12": + return "iPad Pro (12.9-inch) (4th generation)" + case "AppleTV5,3": + return "Apple TV" + case "AppleTV6,2": + return "Apple TV 4K" + case "AudioAccessory1,1": + return "HomePod" + case "AudioAccessory5,1": + return "HomePod mini" + case "i386", "x86_64": + return "Simulator" + default: + return identifier + } +} + +func GetIOSDeviceType(identifier string) string { + if strings.Contains(identifier, "iPhone") { + return "mobile" //"phone" + } + if strings.Contains(identifier, "iPad") { + return "tablet" + } + return "other" +} diff --git a/backend/services/http/ios_device.go b/backend/services/http/ios_device.go deleted file mode 100644 index 2c3474157..000000000 --- a/backend/services/http/ios_device.go +++ /dev/null @@ -1,79 +0,0 @@ -package main - -import ( - "strings" -) - -func MapIOSDevice(identifier string) string { - switch identifier { - case "iPod5,1": return "iPod touch (5th generation)" - case "iPod7,1": return "iPod touch (6th generation)" - case "iPod9,1": return "iPod touch (7th generation)" - case "iPhone3,1", "iPhone3,2", "iPhone3,3": return "iPhone 4" - case "iPhone4,1": return "iPhone 4s" - case "iPhone5,1", "iPhone5,2": return "iPhone 5" - case "iPhone5,3", "iPhone5,4": return "iPhone 5c" - case "iPhone6,1", "iPhone6,2": return "iPhone 5s" - case "iPhone7,2": return "iPhone 6" - case "iPhone7,1": return "iPhone 6 Plus" - case "iPhone8,1": return "iPhone 6s" - case "iPhone8,2": return "iPhone 6s Plus" - case "iPhone8,4": return "iPhone SE" - case "iPhone9,1", "iPhone9,3": return "iPhone 7" - case "iPhone9,2", "iPhone9,4": return "iPhone 7 Plus" - case "iPhone10,1", "iPhone10,4": return "iPhone 8" - case "iPhone10,2", "iPhone10,5": return "iPhone 8 Plus" - case "iPhone10,3", "iPhone10,6": return "iPhone X" - case "iPhone11,2": return "iPhone XS" - case "iPhone11,4", "iPhone11,6": return "iPhone XS Max" - case "iPhone11,8": return "iPhone XR" - case "iPhone12,1": return "iPhone 11" - case "iPhone12,3": return "iPhone 11 Pro" - case "iPhone12,5": return "iPhone 11 Pro Max" - case "iPhone12,8": return "iPhone SE (2nd generation)" - case "iPhone13,1": return "iPhone 12 mini" - case "iPhone13,2": return "iPhone 12" - case "iPhone13,3": return "iPhone 12 Pro" - case "iPhone13,4": return "iPhone 12 Pro Max" - case "iPad2,1", "iPad2,2", "iPad2,3", "iPad2,4":return "iPad 2" - case "iPad3,1", "iPad3,2", "iPad3,3": return "iPad (3rd generation)" - case "iPad3,4", "iPad3,5", "iPad3,6": return "iPad (4th generation)" - case "iPad6,11", "iPad6,12": return "iPad (5th generation)" - case "iPad7,5", "iPad7,6": return "iPad (6th generation)" - case "iPad7,11", "iPad7,12": return "iPad (7th generation)" - case "iPad11,6", "iPad11,7": return "iPad (8th generation)" - case "iPad4,1", "iPad4,2", "iPad4,3": return "iPad Air" - case "iPad5,3", "iPad5,4": return "iPad Air 2" - case "iPad11,3", "iPad11,4": return "iPad Air (3rd generation)" - case "iPad13,1", "iPad13,2": return "iPad Air (4th generation)" - case "iPad2,5", "iPad2,6", "iPad2,7": return "iPad mini" - case "iPad4,4", "iPad4,5", "iPad4,6": return "iPad mini 2" - case "iPad4,7", "iPad4,8", "iPad4,9": return "iPad mini 3" - case "iPad5,1", "iPad5,2": return "iPad mini 4" - case "iPad11,1", "iPad11,2": return "iPad mini (5th generation)" - case "iPad6,3", "iPad6,4": return "iPad Pro (9.7-inch)" - case "iPad7,3", "iPad7,4": return "iPad Pro (10.5-inch)" - case "iPad8,1", "iPad8,2", "iPad8,3", "iPad8,4":return "iPad Pro (11-inch) (1st generation)" - case "iPad8,9", "iPad8,10": return "iPad Pro (11-inch) (2nd generation)" - case "iPad6,7", "iPad6,8": return "iPad Pro (12.9-inch) (1st generation)" - case "iPad7,1", "iPad7,2": return "iPad Pro (12.9-inch) (2nd generation)" - case "iPad8,5", "iPad8,6", "iPad8,7", "iPad8,8":return "iPad Pro (12.9-inch) (3rd generation)" - case "iPad8,11", "iPad8,12": return "iPad Pro (12.9-inch) (4th generation)" - case "AppleTV5,3": return "Apple TV" - case "AppleTV6,2": return "Apple TV 4K" - case "AudioAccessory1,1": return "HomePod" - case "AudioAccessory5,1": return "HomePod mini" - case "i386", "x86_64": return "Simulator" - default: return identifier - } -} - -func GetIOSDeviceType(identifier string) string { - if strings.Contains(identifier, "iPhone") { - return "mobile" //"phone" - } - if strings.Contains(identifier, "iPad") { - return "tablet" - } - return "other" -} \ No newline at end of file diff --git a/backend/services/http/main.go b/backend/services/http/main.go index 8ed8b6d95..f0ae22d32 100644 --- a/backend/services/http/main.go +++ b/backend/services/http/main.go @@ -10,19 +10,17 @@ import ( "golang.org/x/net/http2" - + "openreplay/backend/pkg/db/cache" + "openreplay/backend/pkg/db/postgres" "openreplay/backend/pkg/env" "openreplay/backend/pkg/flakeid" "openreplay/backend/pkg/queue" "openreplay/backend/pkg/queue/types" "openreplay/backend/pkg/storage" - "openreplay/backend/pkg/db/postgres" - "openreplay/backend/pkg/db/cache" - "openreplay/backend/pkg/url/assets" "openreplay/backend/pkg/token" + "openreplay/backend/pkg/url/assets" "openreplay/backend/services/http/geoip" "openreplay/backend/services/http/uaparser" - ) var rewriter *assets.Rewriter @@ -38,6 +36,7 @@ var TOPIC_RAW_WEB string var TOPIC_RAW_IOS string var TOPIC_CACHE string var TOPIC_TRIGGER string + //var TOPIC_ANALYTICS string var CACHE_ASSESTS bool var BEACON_SIZE_LIMIT int64 @@ -53,7 +52,7 @@ func main() { TOPIC_TRIGGER = env.String("TOPIC_TRIGGER") //TOPIC_ANALYTICS = env.String("TOPIC_ANALYTICS") rewriter = assets.NewRewriter(env.String("ASSETS_ORIGIN")) - pgconn = cache.NewPGCache(postgres.NewConn(env.String("POSTGRES_STRING")), 1000 * 60 * 20) + pgconn = cache.NewPGCache(postgres.NewConn(env.String("POSTGRES_STRING")), 1000*60*20) defer pgconn.Close() s3 = storage.NewS3(env.String("AWS_REGION"), env.String("S3_BUCKET_IOS_IMAGES")) tokenizer = token.NewTokenizer(env.String("TOKEN_SECRET")) @@ -70,7 +69,7 @@ func main() { Handler: http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { // TODO: agree with specification - w.Header().Set("Access-Control-Allow-Origin", "*") + w.Header().Set("Access-Control-Allow-Origin", "*") w.Header().Set("Access-Control-Allow-Methods", "POST") w.Header().Set("Access-Control-Allow-Headers", "Content-Type,Authorization") if r.Method == http.MethodOptions { @@ -79,13 +78,12 @@ func main() { return } - log.Printf("Request: %v - %v ", r.Method, r.URL.Path) - + log.Printf("Request: %v - %v ", r.Method, r.URL.Path) switch r.URL.Path { case "/": w.WriteHeader(http.StatusOK) - case "/v1/web/not-started": + case "/v1/web/not-started": switch r.Method { case http.MethodPost: notStartedHandlerWeb(w, r) diff --git a/backend/services/http/project_id.go b/backend/services/http/project_id.go deleted file mode 100644 index 059576fe8..000000000 --- a/backend/services/http/project_id.go +++ /dev/null @@ -1,12 +0,0 @@ -package main - -func decodeProjectID(projectID uint64) uint64 { - if projectID < 0x10000000000000 || projectID >= 0x20000000000000 { - return 0 - } - projectID = (projectID - 0x10000000000000) * 4212451012670231 & 0xfffffffffffff - if projectID > 0xffffffff { - return 0 - } - return projectID -}