Store source metadata for imported documents (#6136)

app/scenes/Document/components/DataLoader.tsx

@@ -5,6 +5,7 @@ import { NavigationNode, TeamPreference } from "@shared/types";
 import { RevisionHelper } from "@shared/utils/RevisionHelper";
 import Document from "~/models/Document";
 import Revision from "~/models/Revision";
+import Error402 from "~/scenes/Error402";
 import Error404 from "~/scenes/Error404";
 import ErrorOffline from "~/scenes/ErrorOffline";
 import useCurrentTeam from "~/hooks/useCurrentTeam";
@@ -12,7 +13,11 @@ import useCurrentUser from "~/hooks/useCurrentUser";
 import usePolicy from "~/hooks/usePolicy";
 import useStores from "~/hooks/useStores";
 import Logger from "~/utils/Logger";
-import { NotFoundError, OfflineError } from "~/utils/errors";
+import {
+  NotFoundError,
+  OfflineError,
+  PaymentRequiredError,
+} from "~/utils/errors";
 import history from "~/utils/history";
 import { matchDocumentEdit, settingsPath } from "~/utils/routeHelpers";
 import Loading from "./Loading";
@@ -195,7 +200,13 @@ function DataLoader({ match, children }: Props) {
   }, [can.read, can.update, document, isEditRoute, comments, team, shares, ui]);
 
   if (error) {
-    return error instanceof OfflineError ? <ErrorOffline /> : <Error404 />;
+    return error instanceof OfflineError ? (
+      <ErrorOffline />
+    ) : error instanceof PaymentRequiredError ? (
+      <Error402 />
+    ) : (
+      <Error404 />
+    );
   }
 
   if (!document || (revisionId && !revision)) {

app/scenes/Error402.tsx

@@ -0,0 +1,26 @@
+import * as React from "react";
+import { useTranslation } from "react-i18next";
+import { useLocation } from "react-router-dom";
+import Empty from "~/components/Empty";
+import Notice from "~/components/Notice";
+import Scene from "~/components/Scene";
+
+const Error402 = () => {
+  const location = useLocation<{ title?: string }>();
+  const { t } = useTranslation();
+  const title = location.state?.title ?? t("Payment Required");
+
+  return (
+    <Scene title={title}>
+      <h1>{title}</h1>
+      <Empty>
+        <Notice>
+          This document cannot be viewed with the current edition. Please
+          upgrade to a paid license to restore access.
+        </Notice>
+      </Empty>
+    </Scene>
+  );
+};
+
+export default Error402;

app/utils/ApiClient.ts

@@ -12,6 +12,7 @@ import {
   NetworkError,
   NotFoundError,
   OfflineError,
+  PaymentRequiredError,
   RateLimitExceededError,
   RequestError,
   ServiceUnavailableError,
@@ -160,6 +161,10 @@ class ApiClient {
       throw new BadRequestError(error.message);
     }
 
+    if (response.status === 402) {
+      throw new PaymentRequiredError(error.message);
+    }
+
     if (response.status === 403) {
       if (error.error === "user_suspended") {
         await stores.auth.logout(false, false);

app/utils/errors.ts

@@ -8,6 +8,8 @@ export class NetworkError extends ExtendableError {}
 
 export class NotFoundError extends ExtendableError {}
 
+export class PaymentRequiredError extends ExtendableError {}
+
 export class OfflineError extends ExtendableError {}
 
 export class ServiceUnavailableError extends ExtendableError {}

server/commands/documentCreator.ts

@@ -1,27 +1,32 @@
 import { Transaction } from "sequelize";
+import { Optional } from "utility-types";
 import { Document, Event, User } from "@server/models";
 import DocumentHelper from "@server/models/helpers/DocumentHelper";
 
-type Props = {
-  id?: string;
-  urlId?: string;
-  title: string;
-  emoji?: string | null;
-  text?: string;
+type Props = Optional<
+  Pick<
+    Document,
+    | "id"
+    | "urlId"
+    | "title"
+    | "text"
+    | "emoji"
+    | "collectionId"
+    | "parentDocumentId"
+    | "importId"
+    | "template"
+    | "fullWidth"
+    | "sourceMetadata"
+    | "editorVersion"
+    | "publishedAt"
+    | "createdAt"
+    | "updatedAt"
+  >
+> & {
   state?: Buffer;
   publish?: boolean;
-  collectionId?: string | null;
-  parentDocumentId?: string | null;
-  importId?: string;
-  publishedAt?: Date;
-  template?: boolean;
   templateDocument?: Document | null;
-  fullWidth?: boolean;
-  createdAt?: Date;
-  updatedAt?: Date;
   user: User;
-  editorVersion?: string;
-  source?: "import";
   ip?: string;
   transaction?: Transaction;
 };
@@ -46,7 +51,7 @@ export default async function documentCreator({
   user,
   editorVersion,
   publishedAt,
-  source,
+  sourceMetadata,
   ip,
   transaction,
 }: Props): Promise<Document> {
@@ -82,6 +87,7 @@ export default async function documentCreator({
       templateId,
       publishedAt,
       importId,
+      sourceMetadata,
       fullWidth: templateDocument ? templateDocument.fullWidth : fullWidth,
       emoji: templateDocument ? templateDocument.emoji : emoji,
       title: DocumentHelper.replaceTemplateVariables(
@@ -112,7 +118,7 @@ export default async function documentCreator({
       teamId: document.teamId,
       actorId: user.id,
       data: {
-        source,
+        source: importId ? "import" : undefined,
         title: document.title,
         templateId,
       },
@@ -137,7 +143,7 @@ export default async function documentCreator({
         teamId: document.teamId,
         actorId: user.id,
         data: {
-          source,
+          source: importId ? "import" : undefined,
           title: document.title,
         },
         ip,

server/commands/documentLoader.ts

@@ -7,6 +7,7 @@ import {
   InvalidRequestError,
   AuthorizationError,
   AuthenticationError,
+  PaymentRequiredError,
 } from "@server/errors";
 import { Collection, Document, Share, User, Team } from "@server/models";
 import { authorize, can } from "@server/policies";
@@ -119,6 +120,10 @@ export default async function loadDocument({
       throw NotFoundError("Document could not be found for shareId");
     }
 
+    if (document.isTrialImport) {
+      throw PaymentRequiredError();
+    }
+
     // If the user has access to read the document, we can just update
     // the last access date and return the document without additional checks.
     const canReadDocument = user && can(user, "read", document);
@@ -202,6 +207,10 @@ export default async function loadDocument({
       user && authorize(user, "read", document);
     }
 
+    if (document.isTrialImport) {
+      throw PaymentRequiredError();
+    }
+
     collection = document.collection;
   }
 

server/errors.ts

@@ -87,6 +87,12 @@ export function InvalidRequestError(message = "Request invalid") {
   });
 }
 
+export function PaymentRequiredError(message = "Payment required") {
+  return httpErrors(402, message, {
+    id: "payment_required",
+  });
+}
+
 export function NotFoundError(message = "Resource not found") {
   return httpErrors(404, message, {
     id: "not_found",

server/migrations/20231111023920-add-source-metadata.js

@@ -0,0 +1,14 @@
+'use strict';
+
+module.exports = {
+  async up (queryInterface, Sequelize) {
+    await queryInterface.addColumn("documents", "sourceMetadata", {
+      type: Sequelize.JSONB,
+      allowNull: true,
+    });
+  },
+
+  async down (queryInterface) {
+    await queryInterface.removeColumn("documents", "sourceMetadata");
+  }
+};

server/models/Document.ts

@@ -31,9 +31,10 @@ import {
   Length as SimpleLength,
   IsNumeric,
   IsDate,
+  AllowNull,
 } from "sequelize-typescript";
 import isUUID from "validator/lib/isUUID";
-import type { NavigationNode } from "@shared/types";
+import type { NavigationNode, SourceMetadata } from "@shared/types";
 import getTasks from "@shared/utils/getTasks";
 import slugify from "@shared/utils/slugify";
 import { SLUG_URL_REGEX } from "@shared/utils/urlHelpers";
@@ -79,6 +80,11 @@ type AdditionalFindOptions = {
     publishedAt: {
       [Op.ne]: null,
     },
+    sourceMetadata: {
+      trial: {
+        [Op.is]: null,
+      },
+    },
   },
 }))
 @Scopes(() => ({
@@ -399,6 +405,10 @@ class Document extends ParanoidModel {
   @Column(DataType.UUID)
   importId: string | null;
 
+  @AllowNull
+  @Column(DataType.JSONB)
+  sourceMetadata: SourceMetadata | null;
+
   @BelongsTo(() => Document, "parentDocumentId")
   parentDocument: Document | null;
 
@@ -545,10 +555,24 @@ class Document extends ParanoidModel {
     return !this.publishedAt;
   }
 
+  /**
+   * Returns the title of the document or a default if the document is untitled.
+   *
+   * @returns boolean
+   */
   get titleWithDefault(): string {
     return this.title || "Untitled";
   }
 
+  /**
+   * Whether this document was imported during a trial period.
+   *
+   * @returns boolean
+   */
+  get isTrialImport() {
+    return !!(this.importId && this.sourceMetadata?.trial);
+  }
+
   /**
    * Get a list of users that have collaborated on this document
    *

server/queues/tasks/ImportJSONTask.ts

@@ -75,11 +75,12 @@ export default class ImportJSONTask extends ImportTask {
           updatedAt: node.updatedAt ? new Date(node.updatedAt) : undefined,
           publishedAt: node.publishedAt ? new Date(node.publishedAt) : null,
           collectionId,
-          sourceId: node.id,
+          externalId: node.id,
+          mimeType: "application/json",
           parentDocumentId: node.parentDocumentId
             ? find(
                 output.documents,
-                (d) => d.sourceId === node.parentDocumentId
+                (d) => d.externalId === node.parentDocumentId
               )?.id
             : null,
           id,
@@ -101,7 +102,7 @@ export default class ImportJSONTask extends ImportTask {
           buffer: () => zipObject.async("nodebuffer"),
           mimeType,
           path: node.key,
-          sourceId: node.id,
+          externalId: node.id,
         });
       });
     }
@@ -132,7 +133,7 @@ export default class ImportJSONTask extends ImportTask {
               )
             : item.collection.description,
         id: collectionId,
-        sourceId: item.collection.id,
+        externalId: item.collection.id,
       });
 
       if (Object.values(item.documents).length) {
@@ -149,7 +150,7 @@ export default class ImportJSONTask extends ImportTask {
     for (const document of output.documents) {
       for (const attachment of output.attachments) {
         const encodedPath = encodeURI(
-          `/api/attachments.redirect?id=${attachment.sourceId}`
+          `/api/attachments.redirect?id=${attachment.externalId}`
         );
 
         document.text = document.text.replace(

server/queues/tasks/ImportMarkdownZipTask.ts

@@ -139,6 +139,7 @@ export default class ImportMarkdownZipTask extends ImportTask {
               collectionId,
               parentDocumentId,
               path: child.path,
+              mimeType: "text/markdown",
             });
 
             await parseNodeChildren(child.children, collectionId, id);

server/queues/tasks/ImportNotionTask.ts

@@ -62,7 +62,7 @@ export default class ImportNotionTask extends ImportTask {
           const id = uuidv4();
           const match = child.title.match(this.NotionUUIDRegex);
           const name = child.title.replace(this.NotionUUIDRegex, "");
-          const sourceId = match ? match[0].trim() : undefined;
+          const externalId = match ? match[0].trim() : undefined;
 
           // If it's not a text file we're going to treat it as an attachment.
           const mimeType = mime.lookup(child.name);
@@ -79,7 +79,7 @@ export default class ImportNotionTask extends ImportTask {
               path: child.path,
               mimeType,
               buffer: () => zipObject.async("nodebuffer"),
-              sourceId,
+              externalId,
             });
             return;
           }
@@ -95,12 +95,12 @@ export default class ImportNotionTask extends ImportTask {
           });
 
           const existingDocumentIndex = output.documents.findIndex(
-            (doc) => doc.sourceId === sourceId
+            (doc) => doc.externalId === externalId
           );
 
           const existingDocument = output.documents[existingDocumentIndex];
 
-          // If there is an existing document with the same sourceId that means
+          // If there is an existing document with the same externalId that means
           // we've already parsed either a folder or a file referencing the same
           // document, as such we should merge.
           if (existingDocument) {
@@ -122,7 +122,8 @@ export default class ImportNotionTask extends ImportTask {
               collectionId,
               parentDocumentId,
               path: child.path,
-              sourceId,
+              mimeType: mimeType || "text/markdown",
+              externalId,
             });
             await parseNodeChildren(child.children, collectionId, id);
           }
@@ -168,13 +169,13 @@ export default class ImportNotionTask extends ImportTask {
       // instead of a relative or absolute URL within the original zip file.
       for (const link of internalLinksInText) {
         const doc = output.documents.find(
-          (doc) => doc.sourceId === link.sourceId
+          (doc) => doc.externalId === link.externalId
         );
 
         if (!doc) {
           Logger.info(
             "task",
-            `Could not find referenced document with sourceId ${link.sourceId}`
+            `Could not find referenced document with externalId ${link.externalId}`
           );
         } else {
           text = text.replace(link.href, `<<${doc.id}>>`);
@@ -188,11 +189,11 @@ export default class ImportNotionTask extends ImportTask {
     for (const node of tree) {
       const match = node.title.match(this.NotionUUIDRegex);
       const name = node.title.replace(this.NotionUUIDRegex, "");
-      const sourceId = match ? match[0].trim() : undefined;
+      const externalId = match ? match[0].trim() : undefined;
       const mimeType = mime.lookup(node.name);
 
       const existingCollectionIndex = output.collections.findIndex(
-        (collection) => collection.sourceId === sourceId
+        (collection) => collection.externalId === externalId
       );
       const existingCollection = output.collections[existingCollectionIndex];
       const collectionId = existingCollection?.id || uuidv4();
@@ -232,7 +233,7 @@ export default class ImportNotionTask extends ImportTask {
           id: collectionId,
           name,
           description,
-          sourceId,
+          externalId,
         });
       }
     }
@@ -254,19 +255,19 @@ export default class ImportNotionTask extends ImportTask {
 
   /**
    * Extracts internal links from a markdown document, taking into account the
-   * sourceId of the document, which is part of the link title.
+   * externalId of the document, which is part of the link title.
    *
    * @param text The markdown text to parse
    * @returns An array of internal links
    */
   private parseInternalLinks(
     text: string
-  ): { title: string; href: string; sourceId: string }[] {
+  ): { title: string; href: string; externalId: string }[] {
     return compact(
       [...text.matchAll(this.NotionLinkRegex)].map((match) => ({
         title: match[1],
         href: match[2],
-        sourceId: match[3],
+        externalId: match[3],
       }))
     );
   }
@@ -294,7 +295,7 @@ export default class ImportNotionTask extends ImportTask {
 
   /**
    * Regex to find markdown links containing ID's that look like UUID's with the
-   * "-"'s removed, Notion's sourceId format.
+   * "-"'s removed, Notion's externalId format.
    */
   private NotionLinkRegex = /\[([^[]+)]\((.*?([0-9a-fA-F]{32})\..*?)\)/g;
 

server/queues/tasks/ImportTask.ts

@@ -1,3 +1,4 @@
+import path from "path";
 import truncate from "lodash/truncate";
 import {
   AttachmentPreset,
@@ -49,7 +50,7 @@ export type StructuredImportData = {
      */
     description?: string | Record<string, any> | null;
     /** Optional id from import source, useful for mapping */
-    sourceId?: string;
+    externalId?: string;
   }[];
   documents: {
     id: string;
@@ -75,8 +76,9 @@ export type StructuredImportData = {
     createdById?: string;
     createdByEmail?: string | null;
     path: string;
+    mimeType: string;
     /** Optional id from import source, useful for mapping */
-    sourceId?: string;
+    externalId?: string;
   }[];
   attachments: {
     id: string;
@@ -85,7 +87,7 @@ export type StructuredImportData = {
     mimeType: string;
     buffer: () => Promise<Buffer>;
     /** Optional id from import source, useful for mapping */
-    sourceId?: string;
+    externalId?: string;
   }[];
 };
 
@@ -428,7 +430,11 @@ export default abstract class ImportTask extends BaseTask<Props> {
 
           const document = await documentCreator({
             ...options,
-            source: "import",
+            sourceMetadata: {
+              fileName: path.basename(item.path),
+              mimeType: item.mimeType,
+              externalId: item.externalId,
+            },
             id: item.id,
             title: item.title,
             text,

server/routes/api/documents/documents.ts

@@ -1326,17 +1326,23 @@ router.post(
     }
 
     const content = await fs.readFile(file.filepath);
+    const fileName = file.originalFilename ?? file.newFilename;
+    const mimeType = file.mimetype ?? "";
+
     const { text, state, title, emoji } = await documentImporter({
       user,
-      fileName: file.originalFilename ?? file.newFilename,
-      mimeType: file.mimetype ?? "",
+      fileName,
+      mimeType,
       content,
       ip: ctx.request.ip,
       transaction,
     });
 
     const document = await documentCreator({
-      source: "import",
+      sourceMetadata: {
+        fileName,
+        mimeType,
+      },
       title,
       emoji,
       text,

shared/i18n/locales/en_US/translation.json

@@ -600,6 +600,7 @@
   "Search documents": "Search documents",
   "No documents found for your filters.": "No documents found for your filters.",
   "You’ve not got any drafts at the moment.": "You’ve not got any drafts at the moment.",
+  "Payment Required": "Payment Required",
   "Not Found": "Not Found",
   "We were unable to find the page you’re looking for. Go to the <2>homepage</2>?": "We were unable to find the page you’re looking for. Go to the <2>homepage</2>?",
   "Offline": "Offline",

shared/types.ts

@@ -123,6 +123,17 @@ export enum UserPreference {
 
 export type UserPreferences = { [key in UserPreference]?: boolean };
 
+export type SourceMetadata = {
+  /** The original source file name. */
+  fileName?: string;
+  /** The original source mime type. */
+  mimeType?: string;
+  /** An ID in the external source. */
+  externalId?: string;
+  /** Whether the item was created through a trial license. */
+  trial?: boolean;
+};
+
 export type CustomTheme = {
   accent: string;
   accentText: string;