diff --git a/server/commands/documentPermanentDeleter.js b/server/commands/documentPermanentDeleter.js index 6408271a8..d060eb93e 100644 --- a/server/commands/documentPermanentDeleter.js +++ b/server/commands/documentPermanentDeleter.js @@ -4,7 +4,7 @@ import { Document, Attachment } from "../models"; import { sequelize } from "../sequelize"; import parseAttachmentIds from "../utils/parseAttachmentIds"; -export async function documentPermanentDeleter(documents: Document[]) { +export default async function documentPermanentDeleter(documents: Document[]) { const activeDocument = documents.find((doc) => !doc.deletedAt); if (activeDocument) { diff --git a/server/commands/documentPermanentDeleter.test.js b/server/commands/documentPermanentDeleter.test.js index a33cd9de0..678c369f1 100644 --- a/server/commands/documentPermanentDeleter.test.js +++ b/server/commands/documentPermanentDeleter.test.js @@ -3,7 +3,7 @@ import { subDays } from "date-fns"; import { Attachment, Document } from "../models"; import { buildAttachment, buildDocument } from "../test/factories"; import { flushdb } from "../test/support"; -import { documentPermanentDeleter } from "./documentPermanentDeleter"; +import documentPermanentDeleter from "./documentPermanentDeleter"; jest.mock("aws-sdk", () => { const mS3 = { deleteObject: jest.fn().mockReturnThis(), promise: jest.fn() }; diff --git a/server/commands/teamPermanentDeleter.js b/server/commands/teamPermanentDeleter.js new file mode 100644 index 000000000..24d8d9544 --- /dev/null +++ b/server/commands/teamPermanentDeleter.js @@ -0,0 +1,177 @@ +// @flow +import Logger from "../logging/logger"; +import { + ApiKey, + Attachment, + AuthenticationProvider, + Collection, + Document, + Event, + FileOperation, + Group, + Team, + NotificationSetting, + User, + UserAuthentication, + Integration, + SearchQuery, + Share, +} from "../models"; +import { sequelize } from "../sequelize"; + +export default async function teamPermanentDeleter(team: Team) { + if (!team.deletedAt) { + throw new Error( + `Cannot permanently delete ${team.id} team. Please delete it and try again.` + ); + } + + Logger.info( + "commands", + `Permanently deleting team ${team.name} (${team.id})` + ); + + const teamId = team.id; + let transaction; + + try { + transaction = await sequelize.transaction(); + + await Attachment.findAllInBatches( + { + where: { + teamId, + }, + limit: 100, + offset: 0, + }, + async (attachments, options) => { + Logger.info( + "commands", + `Deleting attachments ${options.offset} – ${ + options.offset + options.limit + }…` + ); + + await Promise.all( + attachments.map((attachment) => attachment.destroy({ transaction })) + ); + } + ); + + // Destroy user-relation models + await User.findAllInBatches( + { + attributes: ["id"], + where: { + teamId, + }, + limit: 100, + offset: 0, + }, + async (users) => { + const userIds = users.map((user) => user.id); + + await UserAuthentication.destroy({ + where: { userId: userIds }, + force: true, + transaction, + }); + + await ApiKey.destroy({ + where: { userId: userIds }, + force: true, + transaction, + }); + } + ); + + // Destory team-relation models + await AuthenticationProvider.destroy({ + where: { teamId }, + force: true, + transaction, + }); + + // events must be first due to db constraints + await Event.destroy({ + where: { teamId }, + force: true, + transaction, + }); + + await Collection.destroy({ + where: { teamId }, + force: true, + transaction, + }); + + await Document.unscoped().destroy({ + where: { teamId }, + force: true, + transaction, + }); + + await FileOperation.destroy({ + where: { teamId }, + force: true, + transaction, + }); + + await Group.unscoped().destroy({ + where: { teamId }, + force: true, + transaction, + }); + + await Integration.destroy({ + where: { teamId }, + force: true, + transaction, + }); + + await NotificationSetting.destroy({ + where: { teamId }, + force: true, + transaction, + }); + + await SearchQuery.destroy({ + where: { teamId }, + force: true, + transaction, + }); + + await Share.destroy({ + where: { teamId }, + force: true, + transaction, + }); + + await User.destroy({ + where: { teamId }, + force: true, + transaction, + }); + + await team.destroy({ + force: true, + transaction, + }); + + await Event.create( + { + name: "teams.destroy", + modelId: teamId, + }, + { transaction } + ); + + await transaction.commit(); + } catch (err) { + if (transaction) { + await transaction.rollback(); + } + throw err; + } +} diff --git a/server/commands/teamPermanentDeleter.test.js b/server/commands/teamPermanentDeleter.test.js new file mode 100644 index 000000000..91b1b3a51 --- /dev/null +++ b/server/commands/teamPermanentDeleter.test.js @@ -0,0 +1,100 @@ +// @flow +import { subDays } from "date-fns"; +import { Attachment, User, Document, Collection, Team } from "../models"; +import { + buildAttachment, + buildUser, + buildTeam, + buildDocument, +} from "../test/factories"; +import { flushdb } from "../test/support"; +import teamPermanentDeleter from "./teamPermanentDeleter"; + +jest.mock("aws-sdk", () => { + const mS3 = { deleteObject: jest.fn().mockReturnThis(), promise: jest.fn() }; + return { + S3: jest.fn(() => mS3), + Endpoint: jest.fn(), + }; +}); + +beforeEach(() => flushdb()); + +describe("teamPermanentDeleter", () => { + it("should destroy related data", async () => { + const team = await buildTeam({ + deletedAt: subDays(new Date(), 90), + }); + const user = await buildUser({ + teamId: team.id, + }); + await buildDocument({ + teamId: team.id, + userId: user.id, + }); + + await teamPermanentDeleter(team); + + expect(await Team.count()).toEqual(0); + expect(await User.count()).toEqual(0); + expect(await Document.unscoped().count({ paranoid: false })).toEqual(0); + expect(await Collection.unscoped().count({ paranoid: false })).toEqual(0); + }); + + it("should not destroy unrelated data", async () => { + const team = await buildTeam({ + deletedAt: subDays(new Date(), 90), + }); + + await buildUser(); + await buildTeam(); + await buildDocument(); + + await teamPermanentDeleter(team); + + expect(await Team.count()).toEqual(4); // each build command creates a team + expect(await User.count()).toEqual(2); + expect(await Document.unscoped().count({ paranoid: false })).toEqual(1); + expect(await Collection.unscoped().count({ paranoid: false })).toEqual(1); + }); + + it("should destroy attachments", async () => { + const team = await buildTeam({ + deletedAt: subDays(new Date(), 90), + }); + const user = await buildUser({ + teamId: team.id, + }); + const document = await buildDocument({ + teamId: team.id, + userId: user.id, + }); + await buildAttachment({ + teamId: document.teamId, + documentId: document.id, + }); + + await teamPermanentDeleter(team); + + expect(await Team.count()).toEqual(0); + expect(await User.count()).toEqual(0); + expect(await Attachment.count()).toEqual(0); + expect(await Document.unscoped().count({ paranoid: false })).toEqual(0); + expect(await Collection.unscoped().count({ paranoid: false })).toEqual(0); + }); + + it("should error when trying to destroy undeleted team", async () => { + const team = await buildTeam(); + + let error; + try { + await teamPermanentDeleter(team); + } catch (err) { + error = err.message; + } + + expect(error).toEqual( + `Cannot permanently delete ${team.id} team. Please delete it and try again.` + ); + }); +}); diff --git a/server/migrations/20210921031555-missing-cascades.js b/server/migrations/20210921031555-missing-cascades.js new file mode 100644 index 000000000..37d6f731a --- /dev/null +++ b/server/migrations/20210921031555-missing-cascades.js @@ -0,0 +1,78 @@ + +module.exports = { + up: async (queryInterface, Sequelize) => { + let tableName, constraintName; + + tableName = 'collection_users'; + constraintName = 'collection_users_collectionId_fkey'; + await queryInterface.sequelize.query(`alter table "${tableName}" drop constraint "${constraintName}"`) + await queryInterface.sequelize.query( + `alter table "${tableName}" + add constraint "${constraintName}" foreign key("collectionId") references "collections" ("id") + on delete cascade` + ); + + constraintName = 'collection_users_userId_fkey'; + await queryInterface.sequelize.query(`alter table "${tableName}" drop constraint "${constraintName}"`) + await queryInterface.sequelize.query( + `alter table "${tableName}"\ + add constraint "${constraintName}" foreign key("userId") references "users" ("id") + on delete cascade` + ); + + tableName = 'group_users'; + constraintName = 'group_users_groupId_fkey'; + await queryInterface.sequelize.query(`alter table "${tableName}" drop constraint "${constraintName}"`) + await queryInterface.sequelize.query( + `alter table "${tableName}" + add constraint "${constraintName}" foreign key("groupId") references "groups" ("id") + on delete cascade` + ); + + constraintName = 'group_users_userId_fkey'; + await queryInterface.sequelize.query(`alter table "${tableName}" drop constraint "${constraintName}"`) + await queryInterface.sequelize.query( + `alter table "${tableName}" + add constraint "${constraintName}" foreign key("userId") references "users" ("id") + on delete cascade` + ); + }, + + down: async (queryInterface, Sequelize) => { + let tableName, constraintName; + + tableName = 'collection_users'; + constraintName = 'collection_users_collectionId_fkey'; + await queryInterface.sequelize.query(`alter table "${tableName}" drop constraint "${constraintName}"`) + await queryInterface.sequelize.query( + `alter table "${tableName}"\ + add constraint "${constraintName}" foreign key("collectionId") references "collections" ("id") + on delete no action` + ); + + constraintName = 'collection_users_userId_fkey'; + await queryInterface.sequelize.query(`alter table "${tableName}" drop constraint "${constraintName}"`) + await queryInterface.sequelize.query( + `alter table "${tableName}"\ + add constraint "${constraintName}" foreign key("userId") references "users" ("id") + on delete no action` + ); + + tableName = 'group_users'; + constraintName = 'group_users_groupId_fkey'; + await queryInterface.sequelize.query(`alter table "${tableName}" drop constraint "${constraintName}"`) + await queryInterface.sequelize.query( + `alter table "${tableName}" + add constraint "${constraintName}" foreign key("groupId") references "groups" ("id") + on delete no action` + ); + + constraintName = 'group_users_userId_fkey'; + await queryInterface.sequelize.query(`alter table "${tableName}" drop constraint "${constraintName}"`) + await queryInterface.sequelize.query( + `alter table "${tableName}" + add constraint "${constraintName}" foreign key("userId") references "users" ("id") + on delete no action` + ); + }, +}; \ No newline at end of file diff --git a/server/models/Attachment.js b/server/models/Attachment.js index fdb3d779e..44953744f 100644 --- a/server/models/Attachment.js +++ b/server/models/Attachment.js @@ -54,6 +54,22 @@ const Attachment = sequelize.define( } ); +Attachment.findAllInBatches = async ( + query, + callback: (attachments: Array, query: Object) => Promise +) => { + if (!query.offset) query.offset = 0; + if (!query.limit) query.limit = 10; + let results; + + do { + results = await Attachment.findAll(query); + + await callback(results, query); + query.offset += query.limit; + } while (results.length >= query.limit); +}; + Attachment.beforeDestroy(async (model) => { await deleteFromS3(model.key); }); diff --git a/server/models/User.js b/server/models/User.js index f9df293a1..bf059b564 100644 --- a/server/models/User.js +++ b/server/models/User.js @@ -304,6 +304,22 @@ User.getCounts = async function (teamId: string) { }; }; +User.findAllInBatches = async ( + query, + callback: (users: Array, query: Object) => Promise +) => { + if (!query.offset) query.offset = 0; + if (!query.limit) query.limit = 10; + let results; + + do { + results = await User.findAll(query); + + await callback(results, query); + query.offset += query.limit; + } while (results.length >= query.limit); +}; + User.prototype.demote = async function ( teamId: string, to: "member" | "viewer" diff --git a/server/models/View.js b/server/models/View.js index 45e07d65e..4df2262c6 100644 --- a/server/models/View.js +++ b/server/models/View.js @@ -4,26 +4,20 @@ import { USER_PRESENCE_INTERVAL } from "../../shared/constants"; import { User } from "../models"; import { DataTypes, Op, sequelize } from "../sequelize"; -const View = sequelize.define( - "view", - { - id: { - type: DataTypes.UUID, - defaultValue: DataTypes.UUIDV4, - primaryKey: true, - }, - lastEditingAt: { - type: DataTypes.DATE, - }, - count: { - type: DataTypes.INTEGER, - defaultValue: 1, - }, +const View = sequelize.define("view", { + id: { + type: DataTypes.UUID, + defaultValue: DataTypes.UUIDV4, + primaryKey: true, }, - { - classMethods: {}, - } -); + lastEditingAt: { + type: DataTypes.DATE, + }, + count: { + type: DataTypes.INTEGER, + defaultValue: 1, + }, +}); View.associate = (models) => { View.belongsTo(models.Document); diff --git a/server/routes/api/documents.js b/server/routes/api/documents.js index 7ccfc1afe..70eabae55 100644 --- a/server/routes/api/documents.js +++ b/server/routes/api/documents.js @@ -5,7 +5,7 @@ import { subtractDate } from "../../../shared/utils/date"; import documentCreator from "../../commands/documentCreator"; import documentImporter from "../../commands/documentImporter"; import documentMover from "../../commands/documentMover"; -import { documentPermanentDeleter } from "../../commands/documentPermanentDeleter"; +import documentPermanentDeleter from "../../commands/documentPermanentDeleter"; import env from "../../env"; import { NotFoundError, diff --git a/server/routes/api/utils.js b/server/routes/api/utils.js index 96b9e5328..55672b372 100644 --- a/server/routes/api/utils.js +++ b/server/routes/api/utils.js @@ -1,10 +1,11 @@ // @flow import { subDays } from "date-fns"; import Router from "koa-router"; -import { documentPermanentDeleter } from "../../commands/documentPermanentDeleter"; +import documentPermanentDeleter from "../../commands/documentPermanentDeleter"; +import teamPermanentDeleter from "../../commands/teamPermanentDeleter"; import { AuthenticationError } from "../../errors"; import Logger from "../../logging/logger"; -import { Document, FileOperation } from "../../models"; +import { Document, Team, FileOperation } from "../../models"; import { Op } from "../../sequelize"; const router = new Router(); @@ -59,6 +60,25 @@ router.post("utils.gc", async (ctx) => { }) ); + Logger.info( + "utils", + `Permanently destroying upto ${limit} teams older than 30 days…` + ); + + const teams = await Team.findAll({ + where: { + deletedAt: { + [Op.lt]: subDays(new Date(), 30), + }, + }, + paranoid: false, + limit, + }); + + for (const team of teams) { + await teamPermanentDeleter(team); + } + ctx.body = { success: true, };