532 lines
15 KiB
TypeScript
532 lines
15 KiB
TypeScript
import dotenv from 'dotenv'
|
|
import path from 'path'
|
|
dotenv.config({ path: path.resolve(__dirname, '..', '..', '.env.test') })
|
|
|
|
import fs from 'fs'
|
|
import { FastifyInstance } from 'fastify'
|
|
import { randomUUID } from 'crypto'
|
|
import * as tus from 'tus-js-client'
|
|
import { DetailedError } from 'tus-js-client'
|
|
import { CreateBucketCommand, S3Client } from '@aws-sdk/client-s3'
|
|
|
|
import { logger } from '@internal/monitoring'
|
|
import { getServiceKeyUser, getPostgresConnection } from '@internal/database'
|
|
import { getConfig } from '../config'
|
|
import app from '../app'
|
|
import { checkBucketExists } from './common'
|
|
import { Storage, backends, StorageKnexDB } from '../storage'
|
|
|
|
const { serviceKey, tenantId, storageS3Bucket, storageBackendType } = getConfig()
|
|
const oneChunkFile = fs.createReadStream(path.resolve(__dirname, 'assets', 'sadcat.jpg'))
|
|
const localServerAddress = 'http://127.0.0.1:8999'
|
|
|
|
const backend = backends.createStorageBackend(storageBackendType)
|
|
const client = backend.client
|
|
|
|
describe('Tus multipart', () => {
|
|
let db: StorageKnexDB
|
|
let storage: Storage
|
|
let server: FastifyInstance
|
|
let bucketName: string
|
|
|
|
beforeAll(async () => {
|
|
server = await app({
|
|
logger: logger,
|
|
})
|
|
|
|
await server.listen({
|
|
port: 8999,
|
|
})
|
|
|
|
if (client instanceof S3Client) {
|
|
const bucketExists = await checkBucketExists(client, storageS3Bucket)
|
|
|
|
if (!bucketExists) {
|
|
const createBucketCommand = new CreateBucketCommand({
|
|
Bucket: storageS3Bucket,
|
|
})
|
|
await client.send(createBucketCommand)
|
|
}
|
|
}
|
|
})
|
|
|
|
afterAll(async () => {
|
|
await server.close()
|
|
})
|
|
|
|
beforeEach(async () => {
|
|
const superUser = await getServiceKeyUser(tenantId)
|
|
const pg = await getPostgresConnection({
|
|
superUser,
|
|
user: superUser,
|
|
tenantId: tenantId,
|
|
host: 'localhost',
|
|
})
|
|
|
|
db = new StorageKnexDB(pg, {
|
|
host: 'localhost',
|
|
tenantId,
|
|
})
|
|
|
|
bucketName = randomUUID()
|
|
storage = new Storage(backend, db)
|
|
})
|
|
|
|
it('Can upload an asset with the TUS protocol', async () => {
|
|
const objectName = randomUUID() + '-cat.jpeg'
|
|
|
|
const bucket = await storage.createBucket({
|
|
id: bucketName,
|
|
name: bucketName,
|
|
public: true,
|
|
})
|
|
|
|
const result = await new Promise((resolve, reject) => {
|
|
const upload = new tus.Upload(oneChunkFile, {
|
|
endpoint: `${localServerAddress}/upload/resumable`,
|
|
onShouldRetry: () => false,
|
|
uploadDataDuringCreation: false,
|
|
headers: {
|
|
authorization: `Bearer ${serviceKey}`,
|
|
'x-upsert': 'true',
|
|
},
|
|
metadata: {
|
|
bucketName: bucketName,
|
|
objectName: objectName,
|
|
contentType: 'image/jpeg',
|
|
cacheControl: '3600',
|
|
metadata: JSON.stringify({
|
|
test1: 'test1',
|
|
test2: 'test2',
|
|
}),
|
|
},
|
|
onError: function (error) {
|
|
console.log('Failed because: ' + error)
|
|
reject(error)
|
|
},
|
|
onSuccess: () => {
|
|
resolve(true)
|
|
},
|
|
})
|
|
|
|
upload.start()
|
|
})
|
|
|
|
expect(result).toEqual(true)
|
|
|
|
const dbAsset = await storage.from(bucket.id).findObject(objectName, '*')
|
|
expect(dbAsset).toEqual({
|
|
bucket_id: bucket.id,
|
|
created_at: expect.any(Date),
|
|
id: expect.any(String),
|
|
last_accessed_at: expect.any(Date),
|
|
level: 1,
|
|
metadata: {
|
|
cacheControl: 'max-age=3600',
|
|
contentLength: 29526,
|
|
eTag: '"53e1323c929d57b09b95fbe6d531865c-1"',
|
|
httpStatusCode: 200,
|
|
lastModified: expect.any(String),
|
|
mimetype: 'image/jpeg',
|
|
size: 29526,
|
|
},
|
|
user_metadata: {
|
|
test1: 'test1',
|
|
test2: 'test2',
|
|
},
|
|
name: objectName,
|
|
owner: null,
|
|
owner_id: null,
|
|
path_tokens: [objectName],
|
|
updated_at: expect.any(Date),
|
|
version: expect.any(String),
|
|
})
|
|
})
|
|
|
|
describe('TUS Validation', () => {
|
|
it('Cannot upload to a non-existing bucket', async () => {
|
|
const objectName = randomUUID() + '-cat.jpeg'
|
|
|
|
await storage.createBucket({
|
|
id: bucketName,
|
|
name: bucketName,
|
|
public: true,
|
|
fileSizeLimit: '10kb',
|
|
})
|
|
|
|
try {
|
|
await new Promise((resolve, reject) => {
|
|
const upload = new tus.Upload(oneChunkFile, {
|
|
endpoint: `${localServerAddress}/upload/resumable`,
|
|
onShouldRetry: () => false,
|
|
uploadDataDuringCreation: false,
|
|
headers: {
|
|
authorization: `Bearer ${serviceKey}`,
|
|
'x-upsert': 'true',
|
|
},
|
|
metadata: {
|
|
bucketName: 'doesn-exist',
|
|
objectName: objectName,
|
|
contentType: 'image/jpeg',
|
|
cacheControl: '3600',
|
|
},
|
|
onError: function (error) {
|
|
console.log('Failed because: ' + error)
|
|
reject(error)
|
|
},
|
|
onSuccess: () => {
|
|
resolve(true)
|
|
},
|
|
})
|
|
|
|
upload.start()
|
|
})
|
|
|
|
throw Error('it should error with max-size exceeded')
|
|
} catch (e: any) {
|
|
expect(e).toBeInstanceOf(DetailedError)
|
|
|
|
const err = e as DetailedError
|
|
expect(err.originalResponse.getBody()).toEqual('Bucket not found')
|
|
expect(err.originalResponse.getStatus()).toEqual(404)
|
|
}
|
|
})
|
|
|
|
it('Cannot upload an asset that exceed the maximum bucket size', async () => {
|
|
const objectName = randomUUID() + '-cat.jpeg'
|
|
|
|
await storage.createBucket({
|
|
id: bucketName,
|
|
name: bucketName,
|
|
public: true,
|
|
fileSizeLimit: '10kb',
|
|
})
|
|
|
|
try {
|
|
await new Promise((resolve, reject) => {
|
|
const upload = new tus.Upload(oneChunkFile, {
|
|
endpoint: `${localServerAddress}/upload/resumable`,
|
|
onShouldRetry: () => false,
|
|
uploadDataDuringCreation: false,
|
|
headers: {
|
|
authorization: `Bearer ${serviceKey}`,
|
|
'x-upsert': 'true',
|
|
},
|
|
metadata: {
|
|
bucketName: bucketName,
|
|
objectName: objectName,
|
|
contentType: 'image/jpeg',
|
|
cacheControl: '3600',
|
|
},
|
|
onError: function (error) {
|
|
console.log('Failed because: ' + error)
|
|
reject(error)
|
|
},
|
|
onSuccess: () => {
|
|
resolve(true)
|
|
},
|
|
})
|
|
|
|
upload.start()
|
|
})
|
|
|
|
throw Error('it should error with max-size exceeded')
|
|
} catch (e: any) {
|
|
expect(e).toBeInstanceOf(DetailedError)
|
|
|
|
const err = e as DetailedError
|
|
expect(err.originalResponse.getBody()).toEqual('Maximum size exceeded\n')
|
|
expect(err.originalResponse.getStatus()).toEqual(413)
|
|
}
|
|
})
|
|
})
|
|
|
|
describe('Signed Upload URL', () => {
|
|
it('will allow uploading using signed upload url without authorization token', async () => {
|
|
const bucket = await storage.createBucket({
|
|
id: bucketName,
|
|
name: bucketName,
|
|
public: true,
|
|
})
|
|
|
|
const objectName = randomUUID() + '-cat.jpeg'
|
|
|
|
const signedUpload = await storage
|
|
.from(bucketName)
|
|
.signUploadObjectUrl(objectName, `${bucketName}/${objectName}`, 3600)
|
|
|
|
const result = await new Promise((resolve, reject) => {
|
|
const upload = new tus.Upload(oneChunkFile, {
|
|
endpoint: `${localServerAddress}/upload/resumable/sign`,
|
|
onShouldRetry: () => false,
|
|
uploadDataDuringCreation: false,
|
|
headers: {
|
|
'x-signature': signedUpload.token,
|
|
},
|
|
metadata: {
|
|
bucketName: bucketName,
|
|
objectName: objectName,
|
|
contentType: 'image/jpeg',
|
|
cacheControl: '3600',
|
|
metadata: JSON.stringify({
|
|
test1: 'test1',
|
|
test3: 'test3',
|
|
}),
|
|
},
|
|
onError: function (error) {
|
|
console.log('Failed because: ' + error)
|
|
reject(error)
|
|
},
|
|
onSuccess: () => {
|
|
resolve(true)
|
|
},
|
|
})
|
|
|
|
upload.start()
|
|
})
|
|
|
|
expect(result).toEqual(true)
|
|
|
|
const dbAsset = await storage.from(bucket.id).findObject(objectName, '*')
|
|
expect(dbAsset).toEqual({
|
|
bucket_id: bucket.id,
|
|
created_at: expect.any(Date),
|
|
id: expect.any(String),
|
|
level: 1,
|
|
last_accessed_at: expect.any(Date),
|
|
metadata: {
|
|
cacheControl: 'max-age=3600',
|
|
contentLength: 29526,
|
|
eTag: '"53e1323c929d57b09b95fbe6d531865c-1"',
|
|
httpStatusCode: 200,
|
|
lastModified: expect.any(String),
|
|
mimetype: 'image/jpeg',
|
|
size: 29526,
|
|
},
|
|
user_metadata: {
|
|
test1: 'test1',
|
|
test3: 'test3',
|
|
},
|
|
name: objectName,
|
|
owner: null,
|
|
owner_id: null,
|
|
path_tokens: [objectName],
|
|
updated_at: expect.any(Date),
|
|
version: expect.any(String),
|
|
})
|
|
})
|
|
|
|
it('will allow uploading using signed upload url without authorization token, honouring the owner id', async () => {
|
|
const bucket = await storage.createBucket({
|
|
id: bucketName,
|
|
name: bucketName,
|
|
public: true,
|
|
})
|
|
|
|
const objectName = randomUUID() + '-cat.jpeg'
|
|
|
|
const signedUpload = await storage
|
|
.from(bucketName)
|
|
.signUploadObjectUrl(objectName, `${bucketName}/${objectName}`, 3600, 'some-owner-id')
|
|
|
|
const result = await new Promise((resolve, reject) => {
|
|
const upload = new tus.Upload(oneChunkFile, {
|
|
endpoint: `${localServerAddress}/upload/resumable/sign`,
|
|
onShouldRetry: () => false,
|
|
uploadDataDuringCreation: false,
|
|
headers: {
|
|
'x-signature': signedUpload.token,
|
|
},
|
|
metadata: {
|
|
bucketName: bucketName,
|
|
objectName: objectName,
|
|
contentType: 'image/jpeg',
|
|
cacheControl: '3600',
|
|
},
|
|
onError: function (error) {
|
|
console.log('Failed because: ' + error)
|
|
reject(error)
|
|
},
|
|
onSuccess: () => {
|
|
resolve(true)
|
|
},
|
|
})
|
|
|
|
upload.start()
|
|
})
|
|
|
|
expect(result).toEqual(true)
|
|
|
|
const dbAsset = await storage.from(bucket.id).findObject(objectName, '*')
|
|
expect(dbAsset).toEqual({
|
|
bucket_id: bucket.id,
|
|
created_at: expect.any(Date),
|
|
id: expect.any(String),
|
|
last_accessed_at: expect.any(Date),
|
|
level: 1,
|
|
metadata: {
|
|
cacheControl: 'max-age=3600',
|
|
contentLength: 29526,
|
|
eTag: '"53e1323c929d57b09b95fbe6d531865c-1"',
|
|
httpStatusCode: 200,
|
|
lastModified: expect.any(String),
|
|
mimetype: 'image/jpeg',
|
|
size: 29526,
|
|
},
|
|
user_metadata: null,
|
|
name: objectName,
|
|
owner: null,
|
|
owner_id: 'some-owner-id',
|
|
path_tokens: [objectName],
|
|
updated_at: expect.any(Date),
|
|
version: expect.any(String),
|
|
})
|
|
})
|
|
|
|
it('will not allow uploading using signed upload url with an expired token', async () => {
|
|
await storage.createBucket({
|
|
id: bucketName,
|
|
name: bucketName,
|
|
public: true,
|
|
})
|
|
|
|
const objectName = randomUUID() + '-cat.jpeg'
|
|
|
|
const signedUpload = await storage
|
|
.from(bucketName)
|
|
.signUploadObjectUrl(objectName, `${bucketName}/${objectName}`, 1)
|
|
|
|
await new Promise((resolve) => setTimeout(resolve, 2000))
|
|
|
|
try {
|
|
await new Promise((resolve, reject) => {
|
|
const upload = new tus.Upload(oneChunkFile, {
|
|
endpoint: `${localServerAddress}/upload/resumable/sign`,
|
|
onShouldRetry: () => false,
|
|
uploadDataDuringCreation: false,
|
|
headers: {
|
|
'x-signature': signedUpload.token,
|
|
},
|
|
metadata: {
|
|
bucketName: bucketName,
|
|
objectName: objectName,
|
|
contentType: 'image/jpeg',
|
|
cacheControl: '3600',
|
|
},
|
|
onError: function (error) {
|
|
console.log('Failed because: ' + error)
|
|
reject(error)
|
|
},
|
|
onSuccess: () => {
|
|
resolve(true)
|
|
},
|
|
})
|
|
|
|
upload.start()
|
|
})
|
|
|
|
throw new Error('it should error with expired token')
|
|
} catch (e) {
|
|
expect((e as Error).message).not.toEqual('it should error with expired token')
|
|
|
|
const err = e as DetailedError
|
|
expect(err.originalResponse.getBody()).toEqual('jwt expired')
|
|
expect(err.originalResponse.getStatus()).toEqual(400)
|
|
}
|
|
})
|
|
|
|
it('will not allow uploading using signed upload url with an invalid token', async () => {
|
|
await storage.createBucket({
|
|
id: bucketName,
|
|
name: bucketName,
|
|
public: true,
|
|
})
|
|
|
|
const objectName = randomUUID() + '-cat.jpeg'
|
|
|
|
await new Promise((resolve) => setTimeout(resolve, 2000))
|
|
|
|
try {
|
|
await new Promise((resolve, reject) => {
|
|
const upload = new tus.Upload(oneChunkFile, {
|
|
endpoint: `${localServerAddress}/upload/resumable/sign`,
|
|
onShouldRetry: () => false,
|
|
uploadDataDuringCreation: false,
|
|
headers: {
|
|
'x-signature': 'invalid-token',
|
|
},
|
|
metadata: {
|
|
bucketName: bucketName,
|
|
objectName: objectName,
|
|
contentType: 'image/jpeg',
|
|
cacheControl: '3600',
|
|
},
|
|
onError: function (error) {
|
|
console.log('Failed because: ' + error)
|
|
reject(error)
|
|
},
|
|
onSuccess: () => {
|
|
resolve(true)
|
|
},
|
|
})
|
|
|
|
upload.start()
|
|
})
|
|
|
|
throw new Error('it should error with expired token')
|
|
} catch (e) {
|
|
expect((e as Error).message).not.toEqual('it should error with expired token')
|
|
|
|
const err = e as DetailedError
|
|
expect(err.originalResponse.getBody()).toEqual('jwt malformed')
|
|
expect(err.originalResponse.getStatus()).toEqual(400)
|
|
}
|
|
})
|
|
|
|
it('will not allow uploading using signed upload url without a token', async () => {
|
|
await storage.createBucket({
|
|
id: bucketName,
|
|
name: bucketName,
|
|
public: true,
|
|
})
|
|
|
|
const objectName = randomUUID() + '-cat.jpeg'
|
|
|
|
await new Promise((resolve) => setTimeout(resolve, 2000))
|
|
|
|
try {
|
|
await new Promise((resolve, reject) => {
|
|
const upload = new tus.Upload(oneChunkFile, {
|
|
endpoint: `${localServerAddress}/upload/resumable/sign`,
|
|
onShouldRetry: () => false,
|
|
uploadDataDuringCreation: false,
|
|
metadata: {
|
|
bucketName: bucketName,
|
|
objectName: objectName,
|
|
contentType: 'image/jpeg',
|
|
cacheControl: '3600',
|
|
},
|
|
onError: function (error) {
|
|
console.log('Failed because: ' + error)
|
|
reject(error)
|
|
},
|
|
onSuccess: () => {
|
|
resolve(true)
|
|
},
|
|
})
|
|
|
|
upload.start()
|
|
})
|
|
|
|
throw new Error('it should error with expired token')
|
|
} catch (e) {
|
|
expect((e as Error).message).not.toEqual('it should error with expired token')
|
|
|
|
const err = e as DetailedError
|
|
expect(err.originalResponse.getBody()).toEqual('Missing x-signature header')
|
|
expect(err.originalResponse.getStatus()).toEqual(400)
|
|
}
|
|
})
|
|
})
|
|
})
|