feat(api): support email casing for signup

feat(api): support email casing for login
feat(api): support email casing for password-reset
feat(api): support email casing for invitation
feat(DB): lower-case all emails
feat(DB): detect duplicate emails and delete them if possible

api/schemas.py

@@ -11,6 +11,10 @@ def attribute_to_camel_case(snake_str):
     return components[0] + ''.join(x.title() for x in components[1:])
 
 
+def transform_email(email: str) -> str:
+    return email.lower() if isinstance(email, str) else email
+
+
 class _Grecaptcha(BaseModel):
     g_recaptcha_response: Optional[str] = Field(None, alias='g-recaptcha-response')
 
@@ -18,6 +22,7 @@ class _Grecaptcha(BaseModel):
 class UserLoginSchema(_Grecaptcha):
     email: EmailStr = Field(...)
     password: str = Field(...)
+    _transform_email = validator('email', pre=True, allow_reuse=True)(transform_email)
 
 
 class UserSignupSchema(UserLoginSchema):
@@ -31,17 +36,21 @@ class UserSignupSchema(UserLoginSchema):
 
 class EditUserSchema(BaseModel):
     name: Optional[str] = Field(None)
-    email: Optional[str] = Field(None)
+    email: Optional[EmailStr] = Field(None)
     admin: Optional[bool] = Field(False)
     appearance: Optional[dict] = Field({})
 
+    _transform_email = validator('email', pre=True, allow_reuse=True)(transform_email)
+
 
 class EditUserAppearanceSchema(BaseModel):
     appearance: dict = Field(...)
 
 
 class ForgetPasswordPayloadSchema(_Grecaptcha):
-    email: str = Field(...)
+    email: EmailStr = Field(...)
+
+    _transform_email = validator('email', pre=True, allow_reuse=True)(transform_email)
 
 
 class EditUserPasswordSchema(BaseModel):
@@ -70,7 +79,9 @@ class CurrentAPIContext(BaseModel):
 
 class CurrentContext(CurrentAPIContext):
     user_id: int = Field(...)
-    email: str = Field(...)
+    email: EmailStr = Field(...)
+
+    _transform_email = validator('email', pre=True, allow_reuse=True)(transform_email)
 
 
 class AddSlackSchema(BaseModel):
@@ -115,15 +126,19 @@ class CreateEditWebhookSchema(BaseModel):
 class CreateMemberSchema(BaseModel):
     userId: Optional[int] = Field(None)
     name: str = Field(...)
-    email: str = Field(...)
+    email: EmailStr = Field(...)
     admin: bool = Field(False)
 
+    _transform_email = validator('email', pre=True, allow_reuse=True)(transform_email)
+
 
 class EditMemberSchema(BaseModel):
     name: str = Field(...)
-    email: str = Field(...)
+    email: EmailStr = Field(...)
     admin: bool = Field(False)
 
+    _transform_email = validator('email', pre=True, allow_reuse=True)(transform_email)
+
 
 class EditPasswordByInvitationSchema(BaseModel):
     invitation: str = Field(...)
@@ -244,6 +259,8 @@ class EmailPayloadSchema(BaseModel):
     link: str = Field(...)
     message: str = Field(...)
 
+    _transform_email = validator('email', pre=True, allow_reuse=True)(transform_email)
+
 
 class MemberInvitationPayloadSchema(BaseModel):
     auth: str = Field(...)
@@ -252,6 +269,8 @@ class MemberInvitationPayloadSchema(BaseModel):
     client_id: str = Field(...)
     sender_name: str = Field(...)
 
+    _transform_email = validator('email', pre=True, allow_reuse=True)(transform_email)
+
     class Config:
         alias_generator = attribute_to_camel_case
 

ee/scripts/helm/db/init_dbs/postgresql/1.5.4/1.5.4.sql

@@ -6,6 +6,69 @@ SELECT 'v1.5.4-ee'
 $$ LANGUAGE sql IMMUTABLE;
 
 
+-- to detect duplicate users and delete them if possible
+DO
+$$
+    DECLARE
+        duplicate RECORD;
+    BEGIN
+        IF EXISTS(SELECT user_id
+                  FROM users
+                  WHERE lower(email) =
+                        (SELECT LOWER(email)
+                         FROM users AS su
+                         WHERE LOWER(su.email) = LOWER(users.email)
+                           AND su.user_id != users.user_id
+                         LIMIT 1)
+                  ORDER BY LOWER(email)) THEN
+            raise notice 'duplicate users detected';
+            FOR duplicate IN SELECT user_id, email, deleted_at, verified_email, jwt_iat
+                             FROM users
+                             WHERE lower(email) =
+                                   (SELECT LOWER(email)
+                                    FROM users AS su
+                                    WHERE LOWER(su.email) = LOWER(users.email)
+                                      AND su.user_id != users.user_id
+                                    LIMIT 1)
+                             ORDER BY LOWER(email)
+                LOOP
+                    IF duplicate.deleted_at IS NOT NULL OR duplicate.jwt_iat IS NULL THEN
+                        raise notice 'deleting duplicate user: %  %',duplicate.user_id,duplicate.email;
+                        DELETE FROM users WHERE user_id = duplicate.user_id;
+                    END IF;
+                END LOOP;
+            IF EXISTS(SELECT user_id
+                      FROM users
+                      WHERE lower(email) =
+                            (SELECT LOWER(email)
+                             FROM users AS su
+                             WHERE LOWER(su.email) = LOWER(users.email)
+                               AND su.user_id != users.user_id
+                             LIMIT 1)
+                      ORDER BY LOWER(email)) THEN
+                raise notice 'remaining duplicates, please fix (delete) before finishing update';
+                FOR duplicate IN SELECT user_id, email
+                                 FROM users
+                                 WHERE lower(email) =
+                                       (SELECT LOWER(email)
+                                        FROM users AS su
+                                        WHERE LOWER(su.email) = LOWER(users.email)
+                                          AND su.user_id != users.user_id
+                                        LIMIT 1)
+                                 ORDER BY LOWER(email)
+                    LOOP
+                        raise notice 'user: %  %',duplicate.user_id,duplicate.email;
+                    END LOOP;
+                RAISE 'Duplicate users' USING ERRCODE = '42710';
+            END IF;
+        END IF;
+    END;
+$$
+LANGUAGE plpgsql;
+
+UPDATE users
+SET email=LOWER(email);
+
 COMMIT;
 
 CREATE INDEX CONCURRENTLY IF NOT EXISTS autocomplete_value_clickonly_gin_idx ON public.autocomplete USING GIN (value gin_trgm_ops) WHERE type = 'CLICK';

scripts/helm/db/init_dbs/postgresql/1.5.4/1.5.4.sql

@@ -6,6 +6,68 @@ SELECT 'v1.5.4'
 $$ LANGUAGE sql IMMUTABLE;
 
 
+-- to detect duplicate users and delete them if possible
+DO
+$$
+    DECLARE
+        duplicate RECORD;
+    BEGIN
+        IF EXISTS(SELECT user_id
+                  FROM users
+                  WHERE lower(email) =
+                        (SELECT LOWER(email)
+                         FROM users AS su
+                         WHERE LOWER(su.email) = LOWER(users.email)
+                           AND su.user_id != users.user_id
+                         LIMIT 1)
+                  ORDER BY LOWER(email)) THEN
+            raise notice 'duplicate users detected';
+            FOR duplicate IN SELECT user_id, email, deleted_at, verified_email, jwt_iat
+                             FROM users
+                             WHERE lower(email) =
+                                   (SELECT LOWER(email)
+                                    FROM users AS su
+                                    WHERE LOWER(su.email) = LOWER(users.email)
+                                      AND su.user_id != users.user_id
+                                    LIMIT 1)
+                             ORDER BY LOWER(email)
+                LOOP
+                    IF duplicate.deleted_at IS NOT NULL OR duplicate.jwt_iat IS NULL THEN
+                        raise notice 'deleting duplicate user: %  %',duplicate.user_id,duplicate.email;
+                        DELETE FROM users WHERE user_id = duplicate.user_id;
+                    END IF;
+                END LOOP;
+            IF EXISTS(SELECT user_id
+                      FROM users
+                      WHERE lower(email) =
+                            (SELECT LOWER(email)
+                             FROM users AS su
+                             WHERE LOWER(su.email) = LOWER(users.email)
+                               AND su.user_id != users.user_id
+                             LIMIT 1)
+                      ORDER BY LOWER(email)) THEN
+                raise notice 'remaining duplicates, please fix (delete) before finishing update';
+                FOR duplicate IN SELECT user_id, email
+                                 FROM users
+                                 WHERE lower(email) =
+                                       (SELECT LOWER(email)
+                                        FROM users AS su
+                                        WHERE LOWER(su.email) = LOWER(users.email)
+                                          AND su.user_id != users.user_id
+                                        LIMIT 1)
+                                 ORDER BY LOWER(email)
+                    LOOP
+                        raise notice 'user: %  %',duplicate.user_id,duplicate.email;
+                    END LOOP;
+                RAISE 'Duplicate users' USING ERRCODE = '42710';
+            END IF;
+        END IF;
+    END;
+$$
+LANGUAGE plpgsql;
+
+UPDATE users SET email=LOWER(email);
+
 COMMIT;
 
 CREATE INDEX CONCURRENTLY IF NOT EXISTS autocomplete_value_clickonly_gin_idx ON public.autocomplete USING GIN (value gin_trgm_ops) WHERE type = 'CLICK';