Doing so has a number of benefits - Overcome the 10Mb limit of the API gateway the lambdas have to go through - By storing the key as the hash of the code we can return previous generated assets, i.e. caching - cost, transfering assets into the bucket within the AWS ecosystem is faster than return, and there fore the lambdas execute for less time - Sets us up for the future as when generating artifacts for repos when there is a change to master etc we want to store these assets somewhere and s3 is an obvious choice - Solved a weird CORS issue where I couldn't get CORS working with binaryMediaTypes enabled, don't need binary types when dumping in s3 Resolves #316
149 lines
3.7 KiB
JavaScript
149 lines
3.7 KiB
JavaScript
const { exec } = require('child_process')
|
|
const { promises } = require('fs')
|
|
const { writeFile } = promises
|
|
const { createHash } = require('crypto')
|
|
|
|
const CONSOLE_MESSAGE_KEY = 'console-message-b64'
|
|
function putConsoleMessageInMetadata(consoleMessage) {
|
|
return {
|
|
[CONSOLE_MESSAGE_KEY]: Buffer.from(consoleMessage, 'utf-8').toString(
|
|
'base64'
|
|
),
|
|
}
|
|
}
|
|
function getConsoleMessageFromMetadata(metadata) {
|
|
return Buffer.from(metadata[CONSOLE_MESSAGE_KEY], 'base64').toString('utf-8')
|
|
}
|
|
|
|
async function makeFile(file, extension = '.scad', makeHash) {
|
|
const tempFile = 'a' + makeHash() // 'a' ensure nothing funny happens if it start with a bad character like "-", maybe I should pick a safer id generator :shrug:
|
|
console.log(`file to write: ${file}`)
|
|
|
|
await runCommand(`mkdir /tmp/${tempFile}`)
|
|
await writeFile(`/tmp/${tempFile}/main${extension}`, file)
|
|
return tempFile
|
|
}
|
|
|
|
async function runCommand(command, timeout = 5000) {
|
|
return new Promise((resolve, reject) => {
|
|
exec(command, (error, stdout, stderr) => {
|
|
if (error) {
|
|
console.log(`error: ${error.message}`)
|
|
console.log(`stderr: ${stderr}`)
|
|
console.log(`stdout: ${stdout}`)
|
|
reject(stdout || stderr) // it seems random if the message is in stdout or stderr, but not normally both
|
|
return
|
|
}
|
|
if (stderr) {
|
|
console.log(`stderr: ${stderr}`)
|
|
resolve(stderr)
|
|
return
|
|
}
|
|
console.log(`stdout: ${stdout}`)
|
|
resolve(stdout)
|
|
})
|
|
setTimeout(() => {
|
|
reject('timeout')
|
|
}, timeout)
|
|
})
|
|
}
|
|
|
|
function makeHash(script) {
|
|
return createHash('sha256').update(script).digest('hex')
|
|
}
|
|
|
|
async function checkIfAlreadyExists(params, s3) {
|
|
try {
|
|
const objectHead = await s3.headObject(params).promise()
|
|
const consoleMessage = getConsoleMessageFromMetadata(objectHead.Metadata)
|
|
console.log('consoleMessage', consoleMessage)
|
|
return { isAlreadyInBucket: true, consoleMessage }
|
|
} catch (e) {
|
|
console.log("couldn't find it", e)
|
|
return { isAlreadyInBucket: false }
|
|
}
|
|
}
|
|
|
|
function getObjectUrl(params, s3) {
|
|
const HALF_HOUR = 1800
|
|
return s3.getSignedUrl('getObject', {
|
|
...params,
|
|
Expires: HALF_HOUR,
|
|
})
|
|
}
|
|
|
|
function loggerWrap(handler) {
|
|
return (req, _context, callback) => {
|
|
try {
|
|
return handler(req, _context, callback)
|
|
} catch (e) {
|
|
console.log('error in handler', e)
|
|
}
|
|
}
|
|
}
|
|
|
|
async function storeAssetAndReturnUrl({
|
|
error,
|
|
callback,
|
|
fullPath,
|
|
consoleMessage,
|
|
key,
|
|
s3,
|
|
params,
|
|
}) {
|
|
if (error) {
|
|
const response = {
|
|
statusCode: 400,
|
|
body: JSON.stringify({ error, fullPath }),
|
|
}
|
|
callback(null, response)
|
|
return
|
|
} else {
|
|
console.log(`got result in route: ${consoleMessage}, file is: ${fullPath}`)
|
|
const { readFile } = require('fs/promises')
|
|
let buffer
|
|
|
|
try {
|
|
buffer = await readFile(fullPath)
|
|
} catch (e) {
|
|
console.log('read file error', e)
|
|
const response = {
|
|
statusCode: 400,
|
|
body: JSON.stringify({ error: consoleMessage, fullPath }),
|
|
}
|
|
callback(null, response)
|
|
return
|
|
}
|
|
const storedRender = await s3
|
|
.putObject({
|
|
Bucket: process.env.BUCKET,
|
|
Key: key,
|
|
Body: buffer,
|
|
Metadata: putConsoleMessageInMetadata(consoleMessage),
|
|
})
|
|
.promise()
|
|
console.log('stored object', storedRender)
|
|
const url = getObjectUrl(params, s3)
|
|
console.log('url', url)
|
|
const response = {
|
|
statusCode: 200,
|
|
body: JSON.stringify({
|
|
url,
|
|
consoleMessage,
|
|
}),
|
|
}
|
|
callback(null, response)
|
|
return
|
|
}
|
|
}
|
|
|
|
module.exports = {
|
|
runCommand,
|
|
makeFile,
|
|
makeHash,
|
|
checkIfAlreadyExists,
|
|
getObjectUrl,
|
|
loggerWrap,
|
|
storeAssetAndReturnUrl,
|
|
}
|