From 783008bd0fa46bf814cbd550070c1f5c507cf15d Mon Sep 17 00:00:00 2001 From: Taha Yassine Kraiem Date: Fri, 18 Mar 2022 17:05:51 +0100 Subject: [PATCH] feat(api): support email casing for signup feat(api): support email casing for login feat(api): support email casing for password-reset feat(api): support email casing for invitation feat(DB): lower-case all emails feat(DB): detect duplicate emails and delete them if possible --- api/schemas.py | 29 +++++++-- .../db/init_dbs/postgresql/1.5.4/1.5.4.sql | 63 +++++++++++++++++++ .../db/init_dbs/postgresql/1.5.4/1.5.4.sql | 62 ++++++++++++++++++ 3 files changed, 149 insertions(+), 5 deletions(-) diff --git a/api/schemas.py b/api/schemas.py index 3b4fefbd6..77cb78c05 100644 --- a/api/schemas.py +++ b/api/schemas.py @@ -11,6 +11,10 @@ def attribute_to_camel_case(snake_str): return components[0] + ''.join(x.title() for x in components[1:]) +def transform_email(email: str) -> str: + return email.lower() if isinstance(email, str) else email + + class _Grecaptcha(BaseModel): g_recaptcha_response: Optional[str] = Field(None, alias='g-recaptcha-response') @@ -18,6 +22,7 @@ class _Grecaptcha(BaseModel): class UserLoginSchema(_Grecaptcha): email: EmailStr = Field(...) password: str = Field(...) + _transform_email = validator('email', pre=True, allow_reuse=True)(transform_email) class UserSignupSchema(UserLoginSchema): @@ -31,17 +36,21 @@ class UserSignupSchema(UserLoginSchema): class EditUserSchema(BaseModel): name: Optional[str] = Field(None) - email: Optional[str] = Field(None) + email: Optional[EmailStr] = Field(None) admin: Optional[bool] = Field(False) appearance: Optional[dict] = Field({}) + _transform_email = validator('email', pre=True, allow_reuse=True)(transform_email) + class EditUserAppearanceSchema(BaseModel): appearance: dict = Field(...) class ForgetPasswordPayloadSchema(_Grecaptcha): - email: str = Field(...) + email: EmailStr = Field(...) + + _transform_email = validator('email', pre=True, allow_reuse=True)(transform_email) class EditUserPasswordSchema(BaseModel): @@ -70,7 +79,9 @@ class CurrentAPIContext(BaseModel): class CurrentContext(CurrentAPIContext): user_id: int = Field(...) - email: str = Field(...) + email: EmailStr = Field(...) + + _transform_email = validator('email', pre=True, allow_reuse=True)(transform_email) class AddSlackSchema(BaseModel): @@ -115,15 +126,19 @@ class CreateEditWebhookSchema(BaseModel): class CreateMemberSchema(BaseModel): userId: Optional[int] = Field(None) name: str = Field(...) - email: str = Field(...) + email: EmailStr = Field(...) admin: bool = Field(False) + _transform_email = validator('email', pre=True, allow_reuse=True)(transform_email) + class EditMemberSchema(BaseModel): name: str = Field(...) - email: str = Field(...) + email: EmailStr = Field(...) admin: bool = Field(False) + _transform_email = validator('email', pre=True, allow_reuse=True)(transform_email) + class EditPasswordByInvitationSchema(BaseModel): invitation: str = Field(...) @@ -244,6 +259,8 @@ class EmailPayloadSchema(BaseModel): link: str = Field(...) message: str = Field(...) + _transform_email = validator('email', pre=True, allow_reuse=True)(transform_email) + class MemberInvitationPayloadSchema(BaseModel): auth: str = Field(...) @@ -252,6 +269,8 @@ class MemberInvitationPayloadSchema(BaseModel): client_id: str = Field(...) sender_name: str = Field(...) + _transform_email = validator('email', pre=True, allow_reuse=True)(transform_email) + class Config: alias_generator = attribute_to_camel_case diff --git a/ee/scripts/helm/db/init_dbs/postgresql/1.5.4/1.5.4.sql b/ee/scripts/helm/db/init_dbs/postgresql/1.5.4/1.5.4.sql index d043cedcb..476b1ece5 100644 --- a/ee/scripts/helm/db/init_dbs/postgresql/1.5.4/1.5.4.sql +++ b/ee/scripts/helm/db/init_dbs/postgresql/1.5.4/1.5.4.sql @@ -6,6 +6,69 @@ SELECT 'v1.5.4-ee' $$ LANGUAGE sql IMMUTABLE; +-- to detect duplicate users and delete them if possible +DO +$$ + DECLARE + duplicate RECORD; + BEGIN + IF EXISTS(SELECT user_id + FROM users + WHERE lower(email) = + (SELECT LOWER(email) + FROM users AS su + WHERE LOWER(su.email) = LOWER(users.email) + AND su.user_id != users.user_id + LIMIT 1) + ORDER BY LOWER(email)) THEN + raise notice 'duplicate users detected'; + FOR duplicate IN SELECT user_id, email, deleted_at, verified_email, jwt_iat + FROM users + WHERE lower(email) = + (SELECT LOWER(email) + FROM users AS su + WHERE LOWER(su.email) = LOWER(users.email) + AND su.user_id != users.user_id + LIMIT 1) + ORDER BY LOWER(email) + LOOP + IF duplicate.deleted_at IS NOT NULL OR duplicate.jwt_iat IS NULL THEN + raise notice 'deleting duplicate user: % %',duplicate.user_id,duplicate.email; + DELETE FROM users WHERE user_id = duplicate.user_id; + END IF; + END LOOP; + IF EXISTS(SELECT user_id + FROM users + WHERE lower(email) = + (SELECT LOWER(email) + FROM users AS su + WHERE LOWER(su.email) = LOWER(users.email) + AND su.user_id != users.user_id + LIMIT 1) + ORDER BY LOWER(email)) THEN + raise notice 'remaining duplicates, please fix (delete) before finishing update'; + FOR duplicate IN SELECT user_id, email + FROM users + WHERE lower(email) = + (SELECT LOWER(email) + FROM users AS su + WHERE LOWER(su.email) = LOWER(users.email) + AND su.user_id != users.user_id + LIMIT 1) + ORDER BY LOWER(email) + LOOP + raise notice 'user: % %',duplicate.user_id,duplicate.email; + END LOOP; + RAISE 'Duplicate users' USING ERRCODE = '42710'; + END IF; + END IF; + END; +$$ +LANGUAGE plpgsql; + +UPDATE users +SET email=LOWER(email); + COMMIT; CREATE INDEX CONCURRENTLY IF NOT EXISTS autocomplete_value_clickonly_gin_idx ON public.autocomplete USING GIN (value gin_trgm_ops) WHERE type = 'CLICK'; diff --git a/scripts/helm/db/init_dbs/postgresql/1.5.4/1.5.4.sql b/scripts/helm/db/init_dbs/postgresql/1.5.4/1.5.4.sql index e03c8dfc7..deb486fa3 100644 --- a/scripts/helm/db/init_dbs/postgresql/1.5.4/1.5.4.sql +++ b/scripts/helm/db/init_dbs/postgresql/1.5.4/1.5.4.sql @@ -6,6 +6,68 @@ SELECT 'v1.5.4' $$ LANGUAGE sql IMMUTABLE; +-- to detect duplicate users and delete them if possible +DO +$$ + DECLARE + duplicate RECORD; + BEGIN + IF EXISTS(SELECT user_id + FROM users + WHERE lower(email) = + (SELECT LOWER(email) + FROM users AS su + WHERE LOWER(su.email) = LOWER(users.email) + AND su.user_id != users.user_id + LIMIT 1) + ORDER BY LOWER(email)) THEN + raise notice 'duplicate users detected'; + FOR duplicate IN SELECT user_id, email, deleted_at, verified_email, jwt_iat + FROM users + WHERE lower(email) = + (SELECT LOWER(email) + FROM users AS su + WHERE LOWER(su.email) = LOWER(users.email) + AND su.user_id != users.user_id + LIMIT 1) + ORDER BY LOWER(email) + LOOP + IF duplicate.deleted_at IS NOT NULL OR duplicate.jwt_iat IS NULL THEN + raise notice 'deleting duplicate user: % %',duplicate.user_id,duplicate.email; + DELETE FROM users WHERE user_id = duplicate.user_id; + END IF; + END LOOP; + IF EXISTS(SELECT user_id + FROM users + WHERE lower(email) = + (SELECT LOWER(email) + FROM users AS su + WHERE LOWER(su.email) = LOWER(users.email) + AND su.user_id != users.user_id + LIMIT 1) + ORDER BY LOWER(email)) THEN + raise notice 'remaining duplicates, please fix (delete) before finishing update'; + FOR duplicate IN SELECT user_id, email + FROM users + WHERE lower(email) = + (SELECT LOWER(email) + FROM users AS su + WHERE LOWER(su.email) = LOWER(users.email) + AND su.user_id != users.user_id + LIMIT 1) + ORDER BY LOWER(email) + LOOP + raise notice 'user: % %',duplicate.user_id,duplicate.email; + END LOOP; + RAISE 'Duplicate users' USING ERRCODE = '42710'; + END IF; + END IF; + END; +$$ +LANGUAGE plpgsql; + +UPDATE users SET email=LOWER(email); + COMMIT; CREATE INDEX CONCURRENTLY IF NOT EXISTS autocomplete_value_clickonly_gin_idx ON public.autocomplete USING GIN (value gin_trgm_ops) WHERE type = 'CLICK';