h3n3/cadhub
1
0
Fork 0
Code Issues 35 Pull Requests Packages Projects Releases Wiki Activity

Remove s3

Browse Source 
but also upgrade the cad lamdbas to use built javascript files,
allowing us to use typescript, and patching redwood
This commit is contained in:
Kurt Hutten
2021-09-04 06:27:25 +10:00
parent 1c13a38ccb
commit 4a3144d360
28 changed files with 457 additions and 4660 deletions
Download Patch File Download Diff File Expand all files Collapse all files

12
app/api/src/docker/README.md
View File

@@ -1,11 +1,12 @@
# Serverless
We're using the serverless from work for deployment
We're using the serverless framework for deployment
```
sls deploy --stage stagename
yarn rw build api && sls deploy --stage <stagename>
```
But [Kurt Hutten](https://github.com/Irev-Dev) is the only one with credentials for deployment atm, though if you wanted to set your own account you could deploy to that if you wanted to test.
Deploying has `yarn rw build` first because the image uses built js files
## Testing changes locally
@@ -18,17 +19,16 @@ The docker build relies on a git ignored file, the aws-lambda-rie. [Download it]
you will also need to create a .env in `app/api/src/docker/.env` for the following env-vars `DEV_AWS_SECRET_ACCESS_KEY, DEV_AWS_ACCESS_KEY_ID and DEV_BUCKET`. Ask @irev-dev for credentials and he can sort you out.
Then cd into this folder `cd api/src/docker` and:
Run
```bash
docker-compose up --build
yarn cad
```
The first time you run this, it has to build the main image it will take some time, but launching again will be quicker.
After which we'll also spin up a light express server to act as an emulator to transform some the request from the front end into how the lambda's expect them (This emulates the aws-api-gateway which changes tranforms the inbound requests somewhat).
```
yarn install
yarn emulate
yarn aws-emulate
```
You can now add CAD_LAMBDA_BASE_URL="http://localhost:8080" to you .env file and restart your main dev process (`yarn rw dev`) comment that line out if you want to go back to using the aws endpoint (and restart the dev process).

6
app/api/src/docker/aws-emulator.js
View File

@@ -16,10 +16,12 @@ const makeRequest = (route, port) => [
console.log(`making post request to ${port}, ${route}`)
try {
const { data } = await axios.post(invocationURL(port), {
body: JSON.stringify(req.body),
body: Buffer.from(JSON.stringify(req.body)).toString('base64'),
})
res.status(data.statusCode)
res.send(data.body)
res.setHeader('Content-Type', 'application/javascript')
res.setHeader('Content-Encoding', 'gzip')
res.send(Buffer.from(data.body, 'base64'))
} catch (e) {
res.status(500)
res.send()

17
app/api/src/docker/cadquery/Dockerfile
View File

@@ -8,7 +8,7 @@ RUN apt-get update -qq
RUN apt-get install -y wget
# install node14, see comment at the to of node14source_setup.sh
ADD common/node14source_setup.sh /nodesource_setup.sh
ADD src/docker/common/node14source_setup.sh /nodesource_setup.sh
RUN ["chmod", "+x", "/nodesource_setup.sh"]
RUN bash nodesource_setup.sh
RUN apt-get install -y nodejs
@@ -26,13 +26,14 @@ RUN apt-get update && \
# Add the lambda emulator for local dev, (see entrypoint.sh for where it's used),
# I have the file locally (gitignored) to speed up build times (as it downloads everytime),
# but you can use the http version of the below ADD command or download it yourself from that url.
ADD common/aws-lambda-rie /usr/local/bin/aws-lambda-rie
ADD src/docker/common/aws-lambda-rie /usr/local/bin/aws-lambda-rie
# ADD https://github.com/aws/aws-lambda-runtime-interface-emulator/releases/download/v1.0/aws-lambda-rie /usr/local/bin/aws-lambda-rie
RUN ["chmod", "+x", "/usr/local/bin/aws-lambda-rie"]
WORKDIR /var/task/
COPY cadquery/package*.json /var/task/
COPY package*.json /var/task/
RUN npm install
RUN npm install aws-lambda-ric@1.0.0
# Get the distribution copy of cq-cli
@@ -45,9 +46,11 @@ RUN unzip cq-cli-Linux-x86_64.zip
RUN chmod +x cq-cli/cq-cli
RUN echo "cadhub-concat-split" > /var/task/cadhub-concat-split
COPY cadquery/*.js /var/task/
COPY common/*.js /var/common/
COPY common/entrypoint.sh /entrypoint.sh
# using built javascript from dist
# run `yarn rw build` before bulding this image
COPY dist/docker/cadquery/*.js /var/task/js/
COPY dist/docker/common/*.js /var/task/common/
COPY src/docker/common/entrypoint.sh /entrypoint.sh
RUN ["chmod", "+x", "/entrypoint.sh"]
ENTRYPOINT ["sh", "/entrypoint.sh"]
CMD [ "cadquery.stl" ]
CMD [ "js/cadquery.stl" ]

56
app/api/src/docker/cadquery/cadquery.js
View File

@@ -1,56 +0,0 @@
const { runCQ } = require('./runCQ')
const middy = require('middy')
const { cors } = require('middy/middlewares')
const AWS = require('aws-sdk')
const tk = require('timekeeper')
const {
makeHash,
checkIfAlreadyExists,
getObjectUrl,
loggerWrap,
storeAssetAndReturnUrl,
} = require('../common/utils')
const s3 = new AWS.S3()
const stl = async (req, _context, callback) => {
_context.callbackWaitsForEmptyEventLoop = false
const eventBody = req.body
console.log('eventBody', eventBody)
const key = `${makeHash(eventBody)}.stl`
console.log('key', key)
const params = {
Bucket: process.env.BUCKET,
Key: key,
}
const previousAsset = await checkIfAlreadyExists(params, s3)
if (previousAsset.isAlreadyInBucket) {
console.log('already in bucket')
const response = {
statusCode: 200,
body: JSON.stringify({
url: getObjectUrl(params, s3, tk),
}),
}
callback(null, response)
return
}
const { file, settings } = JSON.parse(eventBody)
const { error, consoleMessage, fullPath } = await runCQ({ file, settings })
await storeAssetAndReturnUrl({
error,
callback,
fullPath,
consoleMessage,
key,
s3,
params,
tk,
})
}
module.exports = {
stl: middy(loggerWrap(stl)).use(cors()),
}

23
app/api/src/docker/cadquery/cadquery.ts Normal file
View File

@@ -0,0 +1,23 @@
import { runCQ } from './runCQ'
import middy from 'middy'
import { cors } from 'middy/middlewares'
import { loggerWrap, storeAssetAndReturnUrl } from '../common/utils'
const stl = async (req, _context, callback) => {
_context.callbackWaitsForEmptyEventLoop = false
const eventBody = Buffer.from(req.body, 'base64').toString('ascii')
console.log('eventBody', eventBody)
const { file, settings } = JSON.parse(eventBody)
const { error, consoleMessage, fullPath } = await runCQ({ file, settings })
await storeAssetAndReturnUrl({
error,
callback,
fullPath,
consoleMessage,
})
}
module.exports = {
stl: middy(loggerWrap(stl)).use(cors()),
}

1156
app/api/src/docker/cadquery/package-lock.json generated
View File

File diff suppressed because it is too large Load Diff

18
app/api/src/docker/cadquery/package.json
View File

@@ -1,18 +0,0 @@
{
"name": "openscad-endpoint",
"version": "0.0.1",
"description": "endpoint for openscad",
"main": "index.js",
"author": "Kurt Hutten <kurt@kurthutten.com>",
"license": "",
"dependencies": {
"aws-sdk": "^2.907.0",
"cors": "^2.8.5",
"middy": "^0.36.0",
"nanoid": "^3.1.20",
"timekeeper": "2.2.0"
},
"devDependencies": {
"aws-lambda-ric": "^1.0.0"
}
}

7
app/api/src/docker/cadquery/runCQ.js → app/api/src/docker/cadquery/runCQ.ts
View File

@@ -1,7 +1,7 @@
const { writeFiles, runCommand } = require('../common/utils')
const { nanoid } = require('nanoid')
import { writeFiles, runCommand } from '../common/utils'
import { nanoid } from 'nanoid'
module.exports.runCQ = async ({
export const runCQ = async ({
file,
settings: { deflection = 0.3 } = {},
} = {}) => {
@@ -27,6 +27,7 @@ module.exports.runCQ = async ({
{
file: JSON.stringify({
consoleMessage,
type: 'stl',
}),
fileName: 'metadata.json',
},

59
app/api/src/docker/common/utils.js → app/api/src/docker/common/utils.ts
View File

@@ -2,8 +2,12 @@ const { exec } = require('child_process')
const { promises } = require('fs')
const { writeFile } = promises
const { createHash } = require('crypto')
import { readFile } from 'fs/promises'
async function writeFiles(files = [], tempFile) {
export async function writeFiles(
files: { file: string; fileName: string }[] = [],
tempFile: string
): Promise<string> {
console.log(`file to write: ${files.length}`)
try {
@@ -19,7 +23,7 @@ async function writeFiles(files = [], tempFile) {
return tempFile
}
async function runCommand(command, timeout = 5000) {
export async function runCommand(command, timeout = 5000): Promise<string> {
return new Promise((resolve, reject) => {
exec(command, (error, stdout, stderr) => {
if (error) {
@@ -77,7 +81,7 @@ function getObjectUrl(params, s3, tk) {
)
}
function loggerWrap(handler) {
export function loggerWrap(handler) {
return (req, _context, callback) => {
try {
return handler(req, _context, callback)
@@ -87,15 +91,16 @@ function loggerWrap(handler) {
}
}
async function storeAssetAndReturnUrl({
export async function storeAssetAndReturnUrl({
error,
callback,
fullPath,
consoleMessage,
key,
s3,
params,
tk,
}: {
error: string
callback: Function
fullPath: string
consoleMessage: string
}) {
if (error) {
const response = {
@@ -106,11 +111,10 @@ async function storeAssetAndReturnUrl({
return
} else {
console.log(`got result in route: ${consoleMessage}, file is: ${fullPath}`)
const { readFile } = require('fs/promises')
let buffer
let buffer = ''
try {
buffer = await readFile(fullPath)
buffer = await readFile(fullPath, { encoding: 'base64' })
} catch (e) {
console.log('read file error', e)
const response = {
@@ -120,37 +124,16 @@ async function storeAssetAndReturnUrl({
callback(null, response)
return
}
const FiveDays = 432000
const storedRender = await s3
.putObject({
Bucket: process.env.BUCKET,
Key: key,
Body: buffer,
CacheControl: `max-age=${FiveDays}`, // browser caching to stop downloads of the same part
ContentType: 'text/stl',
ContentEncoding: 'gzip',
})
.promise()
console.log('stored object', storedRender)
const url = getObjectUrl(params, s3, tk)
console.log('url', url)
const response = {
statusCode: 200,
body: JSON.stringify({
url,
}),
body: buffer,
isBase64Encoded: true,
headers: {
'Content-Type': 'application/javascript',
'Content-Encoding': 'gzip',
},
}
callback(null, response)
return
}
}
module.exports = {
runCommand,
writeFiles,
makeHash,
checkIfAlreadyExists,
getObjectUrl,
loggerWrap,
storeAssetAndReturnUrl,
}

24
app/api/src/docker/docker-compose.yml
View File

@@ -2,10 +2,14 @@ services:
openscad-preview:
build:
context: ./
dockerfile: ./openscad/Dockerfile
context: ../../
dockerfile: ./src/docker/openscad/Dockerfile
image: openscad
command: openscad.preview
command: js/openscad.preview
# Adding volumes so that the containers can be restarted for js only changes in local dev
volumes:
- ../../dist/docker/openscad:/var/task/js/
- ../../dist/docker/common:/var/task/common/
ports:
- "5052:8080"
environment:
@@ -15,7 +19,10 @@ services:
openscad-stl:
image: openscad
command: openscad.stl
volumes:
- ../../dist/docker/openscad:/var/task/js/
- ../../dist/docker/common:/var/task/common/
command: js/openscad.stl
ports:
- "5053:8080"
environment:
@@ -25,9 +32,12 @@ services:
cadquery-stl:
build:
context: ./
dockerfile: ./cadquery/Dockerfile
command: cadquery.stl
context: ../../
dockerfile: ./src/docker/cadquery/Dockerfile
volumes:
- ../../dist/docker/cadquery:/var/task/js/
- ../../dist/docker/common:/var/task/common/
command: js/cadquery.stl
ports:
- 5060:8080
environment:

19
app/api/src/docker/openscad/Dockerfile
View File

@@ -14,7 +14,7 @@ RUN apt-get update -qq
RUN apt-get install -y openscad-nightly
# install node14, see comment at the to of node14source_setup.sh
ADD common/node14source_setup.sh /nodesource_setup.sh
ADD src/docker/common/node14source_setup.sh /nodesource_setup.sh
RUN ["chmod", "+x", "/nodesource_setup.sh"]
RUN bash nodesource_setup.sh
RUN apt-get install -y nodejs
@@ -32,13 +32,14 @@ RUN apt-get update && \
# Add the lambda emulator for local dev, (see entrypoint.sh for where it's used),
# I have the file locally (gitignored) to speed up build times (as it downloads everytime),
# but you can use the http version of the below ADD command or download it yourself from that url.
ADD common/aws-lambda-rie /usr/local/bin/aws-lambda-rie
ADD src/docker/common/aws-lambda-rie /usr/local/bin/aws-lambda-rie
# ADD https://github.com/aws/aws-lambda-runtime-interface-emulator/releases/download/v1.0/aws-lambda-rie /usr/local/bin/aws-lambda-rie
RUN ["chmod", "+x", "/usr/local/bin/aws-lambda-rie"]
WORKDIR /var/task/
COPY openscad/package*.json /var/task/
COPY package*.json /var/task/
RUN npm install
RUN npm install aws-lambda-ric@1.0.0
# Install OpenSCAD libraries
# It's experimental, so only adding latest Round-Anything for now
@@ -46,14 +47,16 @@ RUN echo "OPENSCADPATH=/var/task/openscad" >>/etc/profile && \
wget -P /var/task/openscad/ https://github.com/Irev-Dev/Round-Anything/archive/refs/tags/1.0.4.zip && \
unzip /var/task/openscad/1.0.4
# Add our own theming (based on DeepOcean with a different "background" and "opencsg-face-back")
COPY openscad/cadhubtheme.json /usr/share/openscad-nightly/color-schemes/render/
COPY src/docker/openscad/cadhubtheme.json /usr/share/openscad-nightly/color-schemes/render/
RUN echo "cadhub-concat-split" > /var/task/cadhub-concat-split
COPY openscad/*.js /var/task/
COPY common/*.js /var/common/
COPY common/entrypoint.sh /entrypoint.sh
# using built javascript from dist
# run `yarn rw build` before bulding this image
COPY dist/docker/openscad/* /var/task/js/
COPY dist/docker/common/* /var/task/common/
COPY src/docker/common/entrypoint.sh /entrypoint.sh
RUN ["chmod", "+x", "/entrypoint.sh"]
ENTRYPOINT ["sh", "/entrypoint.sh"]
CMD [ "openscad.render" ]
CMD [ "js/openscad.render" ]

129
app/api/src/docker/openscad/openscad.js
View File

@@ -1,129 +0,0 @@
const { runScad, stlExport } = require('./runScad')
const middy = require('middy')
const { cors } = require('middy/middlewares')
const AWS = require('aws-sdk')
const tk = require('timekeeper')
const {
makeHash,
checkIfAlreadyExists,
getObjectUrl,
loggerWrap,
storeAssetAndReturnUrl,
} = require('../common/utils')
const s3 = new AWS.S3()
const openScadStlKey = (eventBody) => {
const { file } = JSON.parse(eventBody)
return `${makeHash(JSON.stringify(file))}.stl`
}
const preview = async (req, _context, callback) => {
_context.callbackWaitsForEmptyEventLoop = false
const eventBody = req.body
console.log('eventBody', eventBody)
const key = `${makeHash(eventBody)}.png`
const stlKey = openScadStlKey(eventBody)
console.log('key', key)
const stlParams = {
Bucket: process.env.BUCKET,
Key: stlKey,
}
const params = {
Bucket: process.env.BUCKET,
Key: key,
}
const [previousAssetStl, previousAssetPng] = await Promise.all([
checkIfAlreadyExists(stlParams, s3),
checkIfAlreadyExists(params, s3),
])
const type = previousAssetStl.isAlreadyInBucket ? 'stl' : 'png'
const previousAsset = previousAssetStl.isAlreadyInBucket
? previousAssetStl
: previousAssetPng
if (previousAsset.isAlreadyInBucket) {
console.log('already in bucket')
const response = {
statusCode: 200,
body: JSON.stringify({
url: getObjectUrl(
{
Bucket: process.env.BUCKET,
Key: previousAssetStl.isAlreadyInBucket ? stlKey : key,
},
s3,
tk
),
type,
}),
}
callback(null, response)
return
}
const { file, settings } = JSON.parse(eventBody)
const { error, consoleMessage, fullPath, customizerPath } = await runScad({
file,
settings,
})
await storeAssetAndReturnUrl({
error,
callback,
fullPath,
consoleMessage,
key,
s3,
params,
tk,
})
}
const stl = async (req, _context, callback) => {
_context.callbackWaitsForEmptyEventLoop = false
const eventBody = req.body
console.log(eventBody, 'eventBody')
const stlKey = openScadStlKey(eventBody)
console.log('key', stlKey)
const params = {
Bucket: process.env.BUCKET,
Key: stlKey,
}
console.log('original params', params)
const previousAsset = await checkIfAlreadyExists(params, s3)
if (previousAsset.isAlreadyInBucket) {
console.log('already in bucket')
const response = {
statusCode: 200,
body: JSON.stringify({
url: getObjectUrl({ ...params }, s3, tk),
}),
}
callback(null, response)
return
}
const { file, settings } = JSON.parse(eventBody)
const { error, consoleMessage, fullPath, customizerPath } = await stlExport({
file,
settings,
})
await storeAssetAndReturnUrl({
error,
callback,
fullPath,
consoleMessage,
key: stlKey,
s3,
params,
tk,
})
}
module.exports = {
stl: middy(loggerWrap(stl)).use(cors()),
preview: middy(loggerWrap(preview)).use(cors()),
}

46
app/api/src/docker/openscad/openscad.ts Normal file
View File

@@ -0,0 +1,46 @@
import { runScad, stlExport } from './runScad'
import middy from 'middy'
import { cors } from 'middy/middlewares'
import { loggerWrap, storeAssetAndReturnUrl } from '../common/utils'
const preview = async (req, _context, callback) => {
_context.callbackWaitsForEmptyEventLoop = false
const eventBody = Buffer.from(req.body, 'base64').toString('ascii')
console.log('eventBody', eventBody)
const { file, settings } = JSON.parse(eventBody)
const { error, consoleMessage, fullPath } = await runScad({
file,
settings,
})
await storeAssetAndReturnUrl({
error,
callback,
fullPath,
consoleMessage,
})
}
const stl = async (req, _context, callback) => {
_context.callbackWaitsForEmptyEventLoop = false
const eventBody = Buffer.from(req.body, 'base64').toString('ascii')
console.log(eventBody, 'eventBody')
const { file, settings } = JSON.parse(eventBody)
const { error, consoleMessage, fullPath } = await stlExport({
file,
settings,
})
await storeAssetAndReturnUrl({
error,
callback,
fullPath,
consoleMessage,
})
}
module.exports = {
stl: middy(loggerWrap(stl)).use(cors()),
preview: middy(loggerWrap(preview)).use(cors()),
}

1156
app/api/src/docker/openscad/package-lock.json generated
View File

File diff suppressed because it is too large Load Diff

18
app/api/src/docker/openscad/package.json
View File

@@ -1,18 +0,0 @@
{
"name": "openscad-endpoint",
"version": "0.0.1",
"description": "endpoint for openscad",
"main": "index.js",
"author": "Kurt Hutten <kurt@kurthutten.com>",
"license": "",
"dependencies": {
"aws-sdk": "^2.907.0",
"cors": "^2.8.5",
"middy": "^0.36.0",
"nanoid": "^3.1.20",
"timekeeper": "2.2.0"
},
"devDependencies": {
"aws-lambda-ric": "^1.0.0"
}
}

17
app/api/src/docker/openscad/runScad.js → app/api/src/docker/openscad/runScad.ts
View File

@@ -1,5 +1,5 @@
const { writeFiles, runCommand } = require('../common/utils')
const { nanoid } = require('nanoid')
import { writeFiles, runCommand } from '../common/utils'
import { nanoid } from 'nanoid'
const { readFile } = require('fs/promises')
const OPENSCAD_COMMON = `xvfb-run --auto-servernum --server-args "-screen 0 1024x768x24" openscad-nightly`
@@ -8,7 +8,7 @@ const OPENSCAD_COMMON = `xvfb-run --auto-servernum --server-args "-screen 0 1024
const cleanOpenScadError = (error) =>
error.replace(/["|']\/tmp\/.+\/main.scad["|']/g, "'main.scad'")
module.exports.runScad = async ({
export const runScad = async ({
file,
settings: {
size: { x = 500, y = 500 } = {},
@@ -19,7 +19,12 @@ module.exports.runScad = async ({
dist = 200,
} = {},
} = {}, // TODO add view settings
} = {}) => {
} = {}): Promise<{
error?: string
consoleMessage?: string
fullPath?: string
customizerPath?: string
}> => {
const tempFile = await writeFiles(
[
{ file, fileName: 'main.scad' },
@@ -62,6 +67,7 @@ module.exports.runScad = async ({
file: JSON.stringify({
customizerParams: params,
consoleMessage,
type: 'png',
}),
fileName: 'metadata.json',
},
@@ -78,7 +84,7 @@ module.exports.runScad = async ({
}
}
module.exports.stlExport = async ({ file, settings: { parameters } } = {}) => {
export const stlExport = async ({ file, settings: { parameters } } = {}) => {
const tempFile = await writeFiles(
[
{ file, fileName: 'main.scad' },
@@ -116,6 +122,7 @@ module.exports.stlExport = async ({ file, settings: { parameters } } = {}) => {
file: JSON.stringify({
customizerParams: params,
consoleMessage,
type: 'stl',
}),
fileName: 'metadata.json',
},

1799
app/api/src/docker/package-lock.json generated
View File

File diff suppressed because it is too large Load Diff

20
app/api/src/docker/package.json
View File

@@ -1,20 +0,0 @@
{
"name": "aws-emulator",
"version": "1.0.0",
"description": "thin layer so that we can use docker lambdas locally",
"scripts": {
"lambdas": "docker-compose up --build",
"emulate": "nodemon ./aws-emulator.js",
"watch": "concurrently \"yarn lambdas\" \"yarn emulate\""
},
"main": "aws-emulator.js",
"dependencies": {
"axios": "^0.21.1",
"cors": "^2.8.5",
"express": "^4.17.1"
},
"devDependencies": {
"concurrently": "^6.0.0",
"nodemon": "^2.0.7"
}
}

22
app/api/src/docker/serverless.yml
View File

@@ -3,7 +3,8 @@ service: cad-lambdas
#app: your-app-name
#org: your-org-name
# plugins:
plugins:
- serverless-binary-cors
# - serverless-offline
# You can pin your service to only deploy with a specific Serverless version
@@ -17,13 +18,18 @@ provider:
images:
# this image is built locally and push to ECR
openscadimage:
path: ./
file: ./openscad/Dockerfile
path: ../../
file: ./src/docker/openscad/Dockerfile
cadqueryimage:
path: ./
file: ./cadquery/Dockerfile
path: ../../
file: ./src/docker/cadquery/Dockerfile
apiGateway:
metrics: true
binaryMediaTypes:
# we need to allow binary types to be able to send back images and stls, but it would be better to be more specific
# ie image/png etc. as */* treats everything as binary including the json body as the input the lambdas
# which mean we need to decode the input bode from base64, but the images break with anything other than */* :(
- '*/*'
# you can overwrite defaults here
# stage: dev
@@ -52,7 +58,7 @@ functions:
image:
name: openscadimage
command:
- openscad.preview
- js/openscad.preview
entryPoint:
- '/entrypoint.sh'
events:
@@ -67,7 +73,7 @@ functions:
image:
name: openscadimage
command:
- openscad.stl
- js/openscad.stl
entryPoint:
- '/entrypoint.sh'
events:
@@ -82,7 +88,7 @@ functions:
image:
name: cadqueryimage
command:
- cadquery.stl
- js/cadquery.stl
entryPoint:
- '/entrypoint.sh'
events: