Files
cadhub/app/api/src/docker/common/utils.js
Kurt Hutten 315492a08a Add s3 integration
Doing so has a number of benefits
- Overcome the 10Mb limit of the API gateway the lambdas have to go
through
- By storing the key as the hash of the code we can return previous
generated assets, i.e. caching
- cost, transfering assets into the bucket within the AWS ecosystem
is faster than return, and there fore the lambdas execute for less time
- Sets us up for the future as when generating artifacts for repos when
there is a change to master etc we want to store these assets somewhere
and s3 is an obvious choice
- Solved a weird CORS issue where I couldn't get CORS working with
binaryMediaTypes enabled, don't need binary types when dumping in s3

Resolves #316
2021-05-18 07:13:08 +10:00

149 lines
3.7 KiB
JavaScript

const { exec } = require('child_process')
const { promises } = require('fs')
const { writeFile } = promises
const { createHash } = require('crypto')
const CONSOLE_MESSAGE_KEY = 'console-message-b64'
function putConsoleMessageInMetadata(consoleMessage) {
return {
[CONSOLE_MESSAGE_KEY]: Buffer.from(consoleMessage, 'utf-8').toString(
'base64'
),
}
}
function getConsoleMessageFromMetadata(metadata) {
return Buffer.from(metadata[CONSOLE_MESSAGE_KEY], 'base64').toString('utf-8')
}
async function makeFile(file, extension = '.scad', makeHash) {
const tempFile = 'a' + makeHash() // 'a' ensure nothing funny happens if it start with a bad character like "-", maybe I should pick a safer id generator :shrug:
console.log(`file to write: ${file}`)
await runCommand(`mkdir /tmp/${tempFile}`)
await writeFile(`/tmp/${tempFile}/main${extension}`, file)
return tempFile
}
async function runCommand(command, timeout = 5000) {
return new Promise((resolve, reject) => {
exec(command, (error, stdout, stderr) => {
if (error) {
console.log(`error: ${error.message}`)
console.log(`stderr: ${stderr}`)
console.log(`stdout: ${stdout}`)
reject(stdout || stderr) // it seems random if the message is in stdout or stderr, but not normally both
return
}
if (stderr) {
console.log(`stderr: ${stderr}`)
resolve(stderr)
return
}
console.log(`stdout: ${stdout}`)
resolve(stdout)
})
setTimeout(() => {
reject('timeout')
}, timeout)
})
}
function makeHash(script) {
return createHash('sha256').update(script).digest('hex')
}
async function checkIfAlreadyExists(params, s3) {
try {
const objectHead = await s3.headObject(params).promise()
const consoleMessage = getConsoleMessageFromMetadata(objectHead.Metadata)
console.log('consoleMessage', consoleMessage)
return { isAlreadyInBucket: true, consoleMessage }
} catch (e) {
console.log("couldn't find it", e)
return { isAlreadyInBucket: false }
}
}
function getObjectUrl(params, s3) {
const HALF_HOUR = 1800
return s3.getSignedUrl('getObject', {
...params,
Expires: HALF_HOUR,
})
}
function loggerWrap(handler) {
return (req, _context, callback) => {
try {
return handler(req, _context, callback)
} catch (e) {
console.log('error in handler', e)
}
}
}
async function storeAssetAndReturnUrl({
error,
callback,
fullPath,
consoleMessage,
key,
s3,
params,
}) {
if (error) {
const response = {
statusCode: 400,
body: JSON.stringify({ error, fullPath }),
}
callback(null, response)
return
} else {
console.log(`got result in route: ${consoleMessage}, file is: ${fullPath}`)
const { readFile } = require('fs/promises')
let buffer
try {
buffer = await readFile(fullPath)
} catch (e) {
console.log('read file error', e)
const response = {
statusCode: 400,
body: JSON.stringify({ error: consoleMessage, fullPath }),
}
callback(null, response)
return
}
const storedRender = await s3
.putObject({
Bucket: process.env.BUCKET,
Key: key,
Body: buffer,
Metadata: putConsoleMessageInMetadata(consoleMessage),
})
.promise()
console.log('stored object', storedRender)
const url = getObjectUrl(params, s3)
console.log('url', url)
const response = {
statusCode: 200,
body: JSON.stringify({
url,
consoleMessage,
}),
}
callback(null, response)
return
}
}
module.exports = {
runCommand,
makeFile,
makeHash,
checkIfAlreadyExists,
getObjectUrl,
loggerWrap,
storeAssetAndReturnUrl,
}