h3n3/cadhub
1
0
Fork 0
Code Issues 35 Pull Requests Packages Projects Releases Wiki Activity

Add customizer support for OpenSCAD

Browse Source 
This also includes sending metadata and part of the concatenated gzip,
not in the s3 metadata as that has a 2kb limit.

Resolves #320
This commit is contained in:
Kurt Hutten
2021-08-25 18:57:28 +10:00
parent 5d79efbf15
commit 87f132a684
18 changed files with 388 additions and 227 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
app/api/src/docker/cadquery/Dockerfile
View File

@@ -43,6 +43,7 @@ RUN wget https://github.com/CadQuery/cq-cli/releases/download/v2.2-beta.2/cq-cli
RUN unzip cq-cli-Linux-x86_64.zip
RUN chmod +x cq-cli/cq-cli
RUN echo "cadhub-concat-split" > /var/task/cadhub-concat-split
COPY cadquery/*.js /var/task/
COPY common/*.js /var/common/

1
app/api/src/docker/cadquery/cadquery.js
View File

@@ -31,7 +31,6 @@ const stl = async (req, _context, callback) => {
statusCode: 200,
body: JSON.stringify({
url: getObjectUrl(params, s3, tk),
consoleMessage: previousAsset.consoleMessage,
}),
}
callback(null, response)

28
app/api/src/docker/cadquery/runCQ.js
View File

@@ -1,24 +1,42 @@
const { makeFile, runCommand } = require('../common/utils')
const { writeFiles, runCommand } = require('../common/utils')
const { nanoid } = require('nanoid')
module.exports.runCQ = async ({
file,
settings: { deflection = 0.3 } = {},
} = {}) => {
const tempFile = await makeFile(file, '.py', nanoid)
const fullPath = `/tmp/${tempFile}/output.stl`
const tempFile = await writeFiles(
[{ file, fileName: 'main.py' }],
'a' + nanoid() // 'a' ensure nothing funny happens if it start with a bad character like "-", maybe I should pick a safer id generator :shrug:
)
const fullPath = `/tmp/${tempFile}/output.gz`
const stlPath = `/tmp/${tempFile}/output.stl`
const command = [
`cq-cli/cq-cli`,
`--codec stl`,
`--infile /tmp/${tempFile}/main.py`,
`--outfile ${fullPath}`,
`--outfile ${stlPath}`,
`--outputopts "deflection:${deflection};angularDeflection:${deflection};"`,
`&& gzip ${fullPath}`,
].join(' ')
console.log('command', command)
try {
const consoleMessage = await runCommand(command, 30000)
await writeFiles(
[
{
file: JSON.stringify({
consoleMessage,
}),
fileName: 'metadata.json',
},
],
tempFile
)
await runCommand(
`cat ${stlPath} /var/task/cadhub-concat-split /tmp/${tempFile}/metadata.json | gzip > ${fullPath}`,
15000
)
return { consoleMessage, fullPath }
} catch (error) {
return { error, fullPath }

41
app/api/src/docker/common/utils.js
View File

@@ -3,24 +3,19 @@ const { promises } = require('fs')
const { writeFile } = promises
const { createHash } = require('crypto')
const CONSOLE_MESSAGE_KEY = 'console-message-b64'
function putConsoleMessageInMetadata(consoleMessage) {
return {
[CONSOLE_MESSAGE_KEY]: Buffer.from(consoleMessage, 'utf-8').toString(
'base64'
),
async function writeFiles(files = [], tempFile) {
console.log(`file to write: ${files.length}`)
try {
await runCommand(`mkdir /tmp/${tempFile}`)
} catch (e) {
//
}
}
function getConsoleMessageFromMetadata(metadata) {
return Buffer.from(metadata[CONSOLE_MESSAGE_KEY], 'base64').toString('utf-8')
}
async function makeFile(file, extension = '.scad', makeHash) {
const tempFile = 'a' + makeHash() // 'a' ensure nothing funny happens if it start with a bad character like "-", maybe I should pick a safer id generator :shrug:
console.log(`file to write: ${file}`)
await runCommand(`mkdir /tmp/${tempFile}`)
await writeFile(`/tmp/${tempFile}/main${extension}`, file)
await Promise.all(
files.map(({ file, fileName }) =>
writeFile(`/tmp/${tempFile}/${fileName}`, file)
)
)
return tempFile
}
@@ -54,10 +49,8 @@ function makeHash(script) {
async function checkIfAlreadyExists(params, s3) {
try {
const objectHead = await s3.headObject(params).promise()
const consoleMessage = getConsoleMessageFromMetadata(objectHead.Metadata)
console.log('consoleMessage', consoleMessage)
return { isAlreadyInBucket: true, consoleMessage }
await s3.headObject(params).promise()
return { isAlreadyInBucket: true }
} catch (e) {
console.log("couldn't find it", e)
return { isAlreadyInBucket: false }
@@ -117,7 +110,7 @@ async function storeAssetAndReturnUrl({
let buffer
try {
buffer = await readFile(`${fullPath}.gz`)
buffer = await readFile(fullPath)
} catch (e) {
console.log('read file error', e)
const response = {
@@ -136,7 +129,6 @@ async function storeAssetAndReturnUrl({
CacheControl: `max-age=${FiveDays}`, // browser caching to stop downloads of the same part
ContentType: 'text/stl',
ContentEncoding: 'gzip',
Metadata: putConsoleMessageInMetadata(consoleMessage),
})
.promise()
console.log('stored object', storedRender)
@@ -146,7 +138,6 @@ async function storeAssetAndReturnUrl({
statusCode: 200,
body: JSON.stringify({
url,
consoleMessage,
}),
}
callback(null, response)
@@ -156,7 +147,7 @@ async function storeAssetAndReturnUrl({
module.exports = {
runCommand,
makeFile,
writeFiles,
makeHash,
checkIfAlreadyExists,
getObjectUrl,

12
app/api/src/docker/openscad/Dockerfile
View File

@@ -6,10 +6,12 @@ ARG DEBIAN_FRONTEND=noninteractive
RUN apt-get update -qq
# double check this below, I'm not sure we need inkscape etc
RUN apt-get -y -qq install software-properties-common dirmngr apt-transport-https lsb-release ca-certificates xvfb imagemagick unzip inkscape
RUN add-apt-repository ppa:openscad/releases
RUN apt-get update -qq
RUN apt-get install -y -qq openscad
RUN apt-get install -y curl wget
RUN touch /etc/apt/sources.list.d/openscad.list
RUN echo "deb https://download.opensuse.org/repositories/home:/t-paul/xUbuntu_20.04/ ./" >> /etc/apt/sources.list.d/openscad.list
RUN wget -qO - https://files.openscad.org/OBS-Repository-Key.pub | apt-key add -
RUN apt-get update -qq
RUN apt-get install -y openscad-nightly
# install node14, see comment at the to of node14source_setup.sh
ADD common/node14source_setup.sh /nodesource_setup.sh
@@ -44,7 +46,9 @@ RUN echo "OPENSCADPATH=/var/task/openscad" >>/etc/profile && \
wget -P /var/task/openscad/ https://github.com/Irev-Dev/Round-Anything/archive/refs/tags/1.0.4.zip && \
unzip /var/task/openscad/1.0.4
# Add our own theming (based on DeepOcean with a different "background" and "opencsg-face-back")
COPY openscad/cadhubtheme.json /usr/share/openscad/color-schemes/render/
COPY openscad/cadhubtheme.json /usr/share/openscad-nightly/color-schemes/render/
RUN echo "cadhub-concat-split" > /var/task/cadhub-concat-split
COPY openscad/*.js /var/task/
COPY common/*.js /var/common/

17
app/api/src/docker/openscad/openscad.js
View File

@@ -57,8 +57,6 @@ const preview = async (req, _context, callback) => {
s3,
tk
),
consoleMessage:
previousAsset.consoleMessage || previousAssetPng.consoleMessage,
type,
}),
}
@@ -67,7 +65,10 @@ const preview = async (req, _context, callback) => {
}
const { file, settings } = JSON.parse(eventBody)
const { error, consoleMessage, fullPath } = await runScad({ file, settings })
const { error, consoleMessage, fullPath, customizerPath } = await runScad({
file,
settings,
})
await storeAssetAndReturnUrl({
error,
callback,
@@ -100,14 +101,16 @@ const stl = async (req, _context, callback) => {
statusCode: 200,
body: JSON.stringify({
url: getObjectUrl({ ...params }, s3, tk),
consoleMessage: previousAsset.consoleMessage,
}),
}
callback(null, response)
return
}
const { file } = JSON.parse(eventBody)
const { error, consoleMessage, fullPath } = await stlExport({ file })
const { file, settings } = JSON.parse(eventBody)
const { error, consoleMessage, fullPath, customizerPath } = await stlExport({
file,
settings,
})
await storeAssetAndReturnUrl({
error,
callback,
@@ -121,6 +124,6 @@ const stl = async (req, _context, callback) => {
}
module.exports = {
stl: middy(stl).use(cors()),
stl: middy(loggerWrap(stl)).use(cors()),
preview: middy(loggerWrap(preview)).use(cors()),
}

101
app/api/src/docker/openscad/runScad.js
View File

@@ -1,7 +1,8 @@
const { makeFile, runCommand } = require('../common/utils')
const { writeFiles, runCommand } = require('../common/utils')
const { nanoid } = require('nanoid')
const { readFile } = require('fs/promises')
const OPENSCAD_COMMON = `xvfb-run --auto-servernum --server-args "-screen 0 1024x768x24" openscad`
const OPENSCAD_COMMON = `xvfb-run --auto-servernum --server-args "-screen 0 1024x768x24" openscad-nightly`
/** Removes our generated/hash filename with just "main.scad", so that it's a nice message in the IDE */
const cleanOpenScadError = (error) =>
@@ -11,6 +12,7 @@ module.exports.runScad = async ({
file,
settings: {
size: { x = 500, y = 500 } = {},
parameters,
camera: {
position = { x: 40, y: 40, z: 40 },
rotation = { x: 55, y: 0, z: 25 },
@@ -18,46 +20,113 @@ module.exports.runScad = async ({
} = {},
} = {}, // TODO add view settings
} = {}) => {
const tempFile = await makeFile(file, '.scad', nanoid)
const tempFile = await writeFiles(
[
{ file, fileName: 'main.scad' },
{
file: JSON.stringify({
parameterSets: { default: parameters },
fileFormatVersion: '1',
}),
fileName: 'params.json',
},
],
'a' + nanoid() // 'a' ensure nothing funny happens if it start with a bad character like "-", maybe I should pick a safer id generator :shrug:
)
const { x: rx, y: ry, z: rz } = rotation
const { x: px, y: py, z: pz } = position
const cameraArg = `--camera=${px},${py},${pz},${rx},${ry},${rz},${dist}`
const fullPath = `/tmp/${tempFile}/output.png`
const fullPath = `/tmp/${tempFile}/output.gz`
const imPath = `/tmp/${tempFile}/output.png`
const customizerPath = `/tmp/${tempFile}/customizer.param`
const command = [
OPENSCAD_COMMON,
`-o ${fullPath}`,
`-o ${customizerPath}`,
`-o ${imPath}`,
`-p /tmp/${tempFile}/params.json -P default`,
cameraArg,
`--imgsize=${x},${y}`,
`--colorscheme CadHub`,
`/tmp/${tempFile}/main.scad`,
`&& gzip ${fullPath}`,
].join(' ')
console.log('command', command)
try {
const consoleMessage = await runCommand(command, 15000)
return { consoleMessage, fullPath }
const params = JSON.parse(
await readFile(customizerPath, { encoding: 'ascii' })
).parameters
await writeFiles(
[
{
file: JSON.stringify({
customizerParams: params,
consoleMessage,
}),
fileName: 'metadata.json',
},
],
tempFile
)
await runCommand(
`cat ${imPath} /var/task/cadhub-concat-split /tmp/${tempFile}/metadata.json | gzip > ${fullPath}`,
15000
)
return { consoleMessage, fullPath, customizerPath }
} catch (dirtyError) {
const error = cleanOpenScadError(dirtyError)
return { error }
return { error: cleanOpenScadError(dirtyError) }
}
}
module.exports.stlExport = async ({ file } = {}) => {
const tempFile = await makeFile(file, '.scad', nanoid)
const fullPath = `/tmp/${tempFile}/output.stl`
module.exports.stlExport = async ({ file, settings: { parameters } } = {}) => {
const tempFile = await writeFiles(
[
{ file, fileName: 'main.scad' },
{
file: JSON.stringify({
parameterSets: { default: parameters },
fileFormatVersion: '1',
}),
fileName: 'params.json',
},
],
'a' + nanoid() // 'a' ensure nothing funny happens if it start with a bad character like "-", maybe I should pick a safer id generator :shrug:
)
const fullPath = `/tmp/${tempFile}/output.gz`
const stlPath = `/tmp/${tempFile}/output.stl`
const customizerPath = `/tmp/${tempFile}/customizer.param`
const command = [
OPENSCAD_COMMON,
`--export-format=binstl`,
`-o ${fullPath}`,
// `--export-format=binstl`,
`-o ${customizerPath}`,
`-o ${stlPath}`,
`-p /tmp/${tempFile}/params.json -P default`,
`/tmp/${tempFile}/main.scad`,
`&& gzip ${fullPath}`,
].join(' ')
try {
// lambda will time out before this, we might need to look at background jobs if we do git integration stl generation
const consoleMessage = await runCommand(command, 60000)
return { consoleMessage, fullPath }
const params = JSON.parse(
await readFile(customizerPath, { encoding: 'ascii' })
).parameters
await writeFiles(
[
{
file: JSON.stringify({
customizerParams: params,
consoleMessage,
}),
fileName: 'metadata.json',
},
],
tempFile
)
await runCommand(
`cat ${stlPath} /var/task/cadhub-concat-split /tmp/${tempFile}/metadata.json | gzip > ${fullPath}`,
15000
)
return { consoleMessage, fullPath, customizerPath }
} catch (error) {
return { error, fullPath }
}