From 70e55a039de9e50c0d66ff402b8441d4cbd91300 Mon Sep 17 00:00:00 2001 From: Kurt Hutten Date: Sat, 3 Jul 2021 08:25:20 +1000 Subject: [PATCH 1/2] Implement gzip compression for cad artifacts The stls from CadQuery and OpenSCAD are not compressed and so we're throwing away bandwidth and taking a performance hit by not gziping. Gzip for s3 basically needs to be gziped before upload and than have 'content-type' : 'text/stl' 'content-encoding' : 'gzip' set. https://stackoverflow.com/questions/8080824/how-to-serve-gzipped-assets-from-amazon-s3 The obvious part that needs to change is putObject in app/api/src/docker/common/utils.js but there might be a few more nuances. resolves #391 --- app/api/src/docker/cadquery/runCQ.js | 9 ++++++- app/api/src/docker/common/utils.js | 4 ++- app/api/src/docker/openscad/runScad.js | 25 +++++++++++++++---- .../helpers/cadPackages/cadQueryController.js | 2 +- 4 files changed, 32 insertions(+), 8 deletions(-) diff --git a/app/api/src/docker/cadquery/runCQ.js b/app/api/src/docker/cadquery/runCQ.js index 98b6b1b..8bf1867 100644 --- a/app/api/src/docker/cadquery/runCQ.js +++ b/app/api/src/docker/cadquery/runCQ.js @@ -7,7 +7,14 @@ module.exports.runCQ = async ({ } = {}) => { const tempFile = await makeFile(file, '.py', nanoid) const fullPath = `/tmp/${tempFile}/output.stl` - const command = `cq-cli/cq-cli --codec stl --infile /tmp/${tempFile}/main.py --outfile ${fullPath} --outputopts "deflection:${deflection};angularDeflection:${deflection};"` + const command = [ + `cq-cli/cq-cli`, + `--codec stl`, + `--infile /tmp/${tempFile}/main.py`, + `--outfile ${fullPath}`, + `--outputopts "deflection:${deflection};angularDeflection:${deflection};"`, + `&& gzip ${fullPath}`, + ].join(' ') console.log('command', command) try { diff --git a/app/api/src/docker/common/utils.js b/app/api/src/docker/common/utils.js index d2be437..0fab7ea 100644 --- a/app/api/src/docker/common/utils.js +++ b/app/api/src/docker/common/utils.js @@ -117,7 +117,7 @@ async function storeAssetAndReturnUrl({ let buffer try { - buffer = await readFile(fullPath) + buffer = await readFile(`${fullPath}.gz`) } catch (e) { console.log('read file error', e) const response = { @@ -134,6 +134,8 @@ async function storeAssetAndReturnUrl({ Key: key, Body: buffer, CacheControl: `max-age=${FiveDays}`, // browser caching to stop downloads of the same part + ContentType: 'text/stl', + ContentEncoding: 'gzip', Metadata: putConsoleMessageInMetadata(consoleMessage), }) .promise() diff --git a/app/api/src/docker/openscad/runScad.js b/app/api/src/docker/openscad/runScad.js index 4a4db3c..6e69122 100644 --- a/app/api/src/docker/openscad/runScad.js +++ b/app/api/src/docker/openscad/runScad.js @@ -1,6 +1,8 @@ const { makeFile, runCommand } = require('../common/utils') const { nanoid } = require('nanoid') +const OPENSCAD_COMMON = `xvfb-run --auto-servernum --server-args "-screen 0 1024x768x24" openscad` + /** Removes our generated/hash filename with just "main.scad", so that it's a nice message in the IDE */ const cleanOpenScadError = (error) => error.replace(/["|']\/tmp\/.+\/main.scad["|']/g, "'main.scad'") @@ -21,7 +23,15 @@ module.exports.runScad = async ({ const { x: px, y: py, z: pz } = position const cameraArg = `--camera=${px},${py},${pz},${rx},${ry},${rz},${dist}` const fullPath = `/tmp/${tempFile}/output.png` - const command = `xvfb-run --auto-servernum --server-args "-screen 0 1024x768x24" openscad -o ${fullPath} ${cameraArg} --imgsize=${x},${y} --colorscheme CadHub /tmp/${tempFile}/main.scad` + const command = [ + OPENSCAD_COMMON, + `-o ${fullPath}`, + cameraArg, + `--imgsize=${x},${y}`, + `--colorscheme CadHub`, + `/tmp/${tempFile}/main.scad`, + `&& gzip ${fullPath}`, + ].join(' ') console.log('command', command) try { @@ -36,12 +46,17 @@ module.exports.runScad = async ({ module.exports.stlExport = async ({ file } = {}) => { const tempFile = await makeFile(file, '.scad', nanoid) const fullPath = `/tmp/${tempFile}/output.stl` + const command = [ + OPENSCAD_COMMON, + `--export-format=binstl`, + `-o ${fullPath}`, + `/tmp/${tempFile}/main.scad`, + `&& gzip ${fullPath}`, + ].join(' ') try { - const consoleMessage = await runCommand( - `xvfb-run --auto-servernum --server-args "-screen 0 1024x768x24" openscad -o ${fullPath} /tmp/${tempFile}/main.scad`, - 60000 // lambda will time out before this, we might need to look at background jobs if we do git integration stl generation - ) + // lambda will time out before this, we might need to look at background jobs if we do git integration stl generation + const consoleMessage = await runCommand(command, 60000) return { consoleMessage, fullPath } } catch (error) { return { error, fullPath } diff --git a/app/web/src/helpers/cadPackages/cadQueryController.js b/app/web/src/helpers/cadPackages/cadQueryController.js index 17fd868..0097483 100644 --- a/app/web/src/helpers/cadPackages/cadQueryController.js +++ b/app/web/src/helpers/cadPackages/cadQueryController.js @@ -9,7 +9,7 @@ import { export const render = async ({ code }) => { const body = JSON.stringify({ settings: { - deflection: 0.2, + deflection: 0.15, }, file: code, }) From f176bbe090ac4761cf2b6374c386dfee2536e062 Mon Sep 17 00:00:00 2001 From: Kurt Hutten Date: Sat, 3 Jul 2021 13:07:32 +1000 Subject: [PATCH 2/2] Use high quality render for CadQuery download --- README.md | 2 + app/.env.defaults | 2 +- .../src/components/EditorMenu/EditorMenu.tsx | 2 + app/web/src/components/EditorMenu/helpers.ts | 47 +++++--- app/web/src/components/IdeEditor/IdeEditor.js | 3 +- .../src/components/IdeWrapper/useRender.ts | 3 +- ...eryController.js => cadQueryController.ts} | 8 +- .../cadPackages/{common.js => common.ts} | 14 ++- .../cadPackages/{index.js => index.ts} | 0 ...cadController.js => openScadController.ts} | 5 +- .../hooks/{useIdeState.js => useIdeState.ts} | 102 ++++++++++++------ .../{DevIdePage.js => DevIdePage.tsx} | 14 ++- 12 files changed, 144 insertions(+), 58 deletions(-) rename app/web/src/helpers/cadPackages/{cadQueryController.js => cadQueryController.ts} (88%) rename app/web/src/helpers/cadPackages/{common.js => common.ts} (79%) rename app/web/src/helpers/cadPackages/{index.js => index.ts} (100%) rename app/web/src/helpers/cadPackages/{openScadController.js => openScadController.ts} (95%) rename app/web/src/helpers/hooks/{useIdeState.js => useIdeState.ts} (71%) rename app/web/src/pages/DevIdePage/{DevIdePage.js => DevIdePage.tsx} (63%) diff --git a/README.md b/README.md index 8bc4688..3f712b7 100644 --- a/README.md +++ b/README.md @@ -37,6 +37,8 @@ yarn rw prisma migrate dev yarn rw prisma db seed ``` +p.s. `yarn rw prisma studio` spins up an app to inspect the db + ### Fire up dev ```terminal yarn rw dev diff --git a/app/.env.defaults b/app/.env.defaults index b8293a4..f698efd 100644 --- a/app/.env.defaults +++ b/app/.env.defaults @@ -22,7 +22,7 @@ CLOUDINARY_API_KEY=476712943135152 # EMAIL_PASSWORD=abc123 -CAD_LAMBDA_BASE_URL="http://localhost:8080" +# CAD_LAMBDA_BASE_URL="http://localhost:8080" # sentry GITHUB_ASSIST_APP_ID=23342 diff --git a/app/web/src/components/EditorMenu/EditorMenu.tsx b/app/web/src/components/EditorMenu/EditorMenu.tsx index 7e090c6..391dbd2 100644 --- a/app/web/src/components/EditorMenu/EditorMenu.tsx +++ b/app/web/src/components/EditorMenu/EditorMenu.tsx @@ -10,7 +10,9 @@ const EditorMenu = () => { const { state, thunkDispatch } = useIdeContext() const handleStlDownload = makeStlDownloadHandler({ type: state.objectData?.type, + ideType: state.ideType, geometry: state.objectData?.data, + quality: state.objectData?.quality, fileName: PullTitleFromFirstLine(state.code || ''), thunkDispatch, }) diff --git a/app/web/src/components/EditorMenu/helpers.ts b/app/web/src/components/EditorMenu/helpers.ts index e5443aa..7148971 100644 --- a/app/web/src/components/EditorMenu/helpers.ts +++ b/app/web/src/components/EditorMenu/helpers.ts @@ -2,7 +2,7 @@ import { flow, identity } from 'lodash/fp' import { fileSave } from 'browser-fs-access' import { MeshBasicMaterial, Mesh, Scene } from 'three' import { STLExporter } from 'three/examples/jsm/exporters/STLExporter' -import { requestRender } from 'src/helpers/hooks/useIdeState' +import { requestRender, State } from 'src/helpers/hooks/useIdeState' export const PullTitleFromFirstLine = (code = '') => { const firstLine = code.split('\n').filter(identity)[0] || '' @@ -16,8 +16,24 @@ export const PullTitleFromFirstLine = (code = '') => { ) } +interface makeStlDownloadHandlerArgs { + geometry: any + fileName: string + type: State['objectData']['type'] + ideType: State['ideType'] + thunkDispatch: (a: any) => any + quality: State['objectData']['quality'] +} + export const makeStlDownloadHandler = - ({ geometry, fileName, type, thunkDispatch }) => + ({ + geometry, + fileName, + type, + thunkDispatch, + quality, + ideType, + }: makeStlDownloadHandlerArgs) => () => { const makeStlBlobFromGeo = flow( (geo) => new Mesh(geo, new MeshBasicMaterial()), @@ -36,22 +52,25 @@ export const makeStlDownloadHandler = }) } if (geometry) { - if (type === 'geometry') { + if ( + type === 'geometry' && + (quality === 'high' || ideType === 'openScad') + ) { saveFile(geometry) } else { thunkDispatch((dispatch, getState) => { const state = getState() - if (state.ideType === 'openScad') { - dispatch({ type: 'setLoading' }) - requestRender({ - state, - dispatch, - code: state.code, - viewerSize: state.viewerSize, - camera: state.camera, - specialCadProcess: 'stl', - }).then((result) => result && saveFile(result.data)) - } + const specialCadProcess = ideType === 'openScad' && 'stl' + dispatch({ type: 'setLoading' }) + requestRender({ + state, + dispatch, + code: state.code, + viewerSize: state.viewerSize, + camera: state.camera, + quality: 'high', + specialCadProcess, + }).then((result) => result && saveFile(result.data)) }) } } diff --git a/app/web/src/components/IdeEditor/IdeEditor.js b/app/web/src/components/IdeEditor/IdeEditor.js index 8dc18c6..34e8d04 100644 --- a/app/web/src/components/IdeEditor/IdeEditor.js +++ b/app/web/src/components/IdeEditor/IdeEditor.js @@ -1,7 +1,6 @@ import { useEffect, useState } from 'react' import { useIdeContext } from 'src/helpers/hooks/useIdeContext' -import { makeCodeStoreKey } from 'src/helpers/hooks/useIdeState' -import { requestRender } from 'src/helpers/hooks/useIdeState' +import { makeCodeStoreKey, requestRender } from 'src/helpers/hooks/useIdeState' import Editor, { useMonaco } from '@monaco-editor/react' import { theme } from 'src/../tailwind.config' diff --git a/app/web/src/components/IdeWrapper/useRender.ts b/app/web/src/components/IdeWrapper/useRender.ts index 86ae5a4..bc82f81 100644 --- a/app/web/src/components/IdeWrapper/useRender.ts +++ b/app/web/src/components/IdeWrapper/useRender.ts @@ -1,5 +1,4 @@ -import { makeCodeStoreKey } from 'src/helpers/hooks/useIdeState' -import { requestRender } from 'src/helpers/hooks/useIdeState' +import { makeCodeStoreKey, requestRender } from 'src/helpers/hooks/useIdeState' import { useIdeContext } from 'src/helpers/hooks/useIdeContext' export const useRender = () => { diff --git a/app/web/src/helpers/cadPackages/cadQueryController.js b/app/web/src/helpers/cadPackages/cadQueryController.ts similarity index 88% rename from app/web/src/helpers/cadPackages/cadQueryController.js rename to app/web/src/helpers/cadPackages/cadQueryController.ts index 0097483..943bddc 100644 --- a/app/web/src/helpers/cadPackages/cadQueryController.js +++ b/app/web/src/helpers/cadPackages/cadQueryController.ts @@ -4,12 +4,16 @@ import { createHealthyResponse, createUnhealthyResponse, timeoutErrorMessage, + RenderArgs, } from './common' -export const render = async ({ code }) => { +export const render = async ({ + code, + settings: { quality = 'low' }, +}: RenderArgs) => { const body = JSON.stringify({ settings: { - deflection: 0.15, + deflection: quality === 'low' ? 0.35 : 0.11, }, file: code, }) diff --git a/app/web/src/helpers/cadPackages/common.js b/app/web/src/helpers/cadPackages/common.ts similarity index 79% rename from app/web/src/helpers/cadPackages/common.js rename to app/web/src/helpers/cadPackages/common.ts index 08e68d5..68ffaee 100644 --- a/app/web/src/helpers/cadPackages/common.js +++ b/app/web/src/helpers/cadPackages/common.ts @@ -1,14 +1,24 @@ import { STLLoader } from 'three/examples/jsm/loaders/STLLoader' +import { State } from 'src/helpers/hooks/useIdeState' export const lambdaBaseURL = - // process.env.CAD_LAMBDA_BASE_URL || - 'https://oxt2p7ddgj.execute-api.us-east-1.amazonaws.com/prod' + process.env.CAD_LAMBDA_BASE_URL || + 'https://2inlbple1b.execute-api.us-east-1.amazonaws.com/prod2' export const stlToGeometry = (url) => new Promise((resolve, reject) => { new STLLoader().load(url, resolve, null, reject) }) +export interface RenderArgs { + code: State['code'] + settings: { + camera: State['camera'] + viewerSize: State['viewerSize'] + quality: State['objectData']['quality'] + } +} + export function createHealthyResponse({ date, data, consoleMessage, type }) { return { status: 'healthy', diff --git a/app/web/src/helpers/cadPackages/index.js b/app/web/src/helpers/cadPackages/index.ts similarity index 100% rename from app/web/src/helpers/cadPackages/index.js rename to app/web/src/helpers/cadPackages/index.ts diff --git a/app/web/src/helpers/cadPackages/openScadController.js b/app/web/src/helpers/cadPackages/openScadController.ts similarity index 95% rename from app/web/src/helpers/cadPackages/openScadController.js rename to app/web/src/helpers/cadPackages/openScadController.ts index 80e1587..32b81a0 100644 --- a/app/web/src/helpers/cadPackages/openScadController.js +++ b/app/web/src/helpers/cadPackages/openScadController.ts @@ -4,9 +4,10 @@ import { createHealthyResponse, createUnhealthyResponse, timeoutErrorMessage, + RenderArgs, } from './common' -export const render = async ({ code, settings }) => { +export const render = async ({ code, settings }: RenderArgs) => { const pixelRatio = window.devicePixelRatio || 1 const size = { x: Math.round(settings.viewerSize?.width * pixelRatio), @@ -67,7 +68,7 @@ export const render = async ({ code, settings }) => { } } -export const stl = async ({ code, settings }) => { +export const stl = async ({ code, settings }: RenderArgs) => { const body = JSON.stringify({ settings: {}, file: code, diff --git a/app/web/src/helpers/hooks/useIdeState.js b/app/web/src/helpers/hooks/useIdeState.ts similarity index 71% rename from app/web/src/helpers/hooks/useIdeState.js rename to app/web/src/helpers/hooks/useIdeState.ts index 55f3055..e0b21be 100644 --- a/app/web/src/helpers/hooks/useIdeState.js +++ b/app/web/src/helpers/hooks/useIdeState.ts @@ -43,36 +43,64 @@ show_object(result) const codeStorageKey = 'Last-editor-code' export const makeCodeStoreKey = (ideType) => `${codeStorageKey}-${ideType}` -let mutableState = null +let mutableState: State = null -export const useIdeState = () => { - const code = '' - const initialLayout = { - direction: 'row', - first: 'Editor', - second: { - direction: 'column', - first: 'Viewer', - second: 'Console', - splitPercentage: 70, - }, +interface XYZ { + x: number + y: number + z: number +} + +export interface State { + ideType: 'INIT' | 'openScad' | 'cadQuery' + consoleMessages: { type: 'message' | 'error'; message: string; time: Date }[] + code: string + objectData: { + type: 'INIT' | 'stl' | 'png' | 'geometry' + data: any + quality: 'low' | 'high' } - const initialState = { - ideType: 'INIT', - consoleMessages: [ - { type: 'message', message: 'Initialising', time: new Date() }, - ], - code, - objectData: { - type: 'INIT', - data: null, - }, - layout: initialLayout, - camera: {}, - viewerSize: { width: 0, height: 0 }, - isLoading: false, + layout: any + camera: { + dist?: number + position?: XYZ + rotation?: XYZ } - const reducer = (state, { type, payload }) => { + viewerSize: { width: number; height: number } + isLoading: boolean +} + +const code = '' +const initialLayout = { + direction: 'row', + first: 'Editor', + second: { + direction: 'column', + first: 'Viewer', + second: 'Console', + splitPercentage: 70, + }, +} + +export const initialState: State = { + ideType: 'INIT', + consoleMessages: [ + { type: 'message', message: 'Initialising', time: new Date() }, + ], + code, + objectData: { + type: 'INIT', + data: null, + quality: 'low', + }, + layout: initialLayout, + camera: {}, + viewerSize: { width: 0, height: 0 }, + isLoading: false, +} + +export const useIdeState = (): [State, (actionOrThunk: any) => any] => { + const reducer = (state: State, { type, payload }): State => { switch (type) { case 'initIde': return { @@ -89,6 +117,7 @@ export const useIdeState = () => { return { ...state, objectData: { + ...state.objectData, type: payload.objectData?.type, data: payload.objectData?.data, }, @@ -142,8 +171,19 @@ export const useIdeState = () => { const [state, dispatch] = useReducer(reducer, initialState) mutableState = state - const getState = () => mutableState - return [state, withThunk(dispatch, getState)] + const getState = (): State => mutableState + const thunkDispatch = withThunk(dispatch, getState) + return [state, thunkDispatch] +} + +interface RequestRenderArgs { + state: State + dispatch: any + code: State['code'] + camera: State['camera'] + viewerSize: State['viewerSize'] + quality: State['objectData']['quality'] + specialCadProcess?: string } export const requestRender = ({ @@ -152,8 +192,9 @@ export const requestRender = ({ code, camera, viewerSize, + quality, specialCadProcess = null, -}) => { +}: RequestRenderArgs) => { if ( state.ideType !== 'INIT' && (!state.isLoading || state.objectData?.type === 'INIT') @@ -166,6 +207,7 @@ export const requestRender = ({ settings: { camera, viewerSize, + quality, }, }) .then(({ objectData, message, status }) => { diff --git a/app/web/src/pages/DevIdePage/DevIdePage.js b/app/web/src/pages/DevIdePage/DevIdePage.tsx similarity index 63% rename from app/web/src/pages/DevIdePage/DevIdePage.js rename to app/web/src/pages/DevIdePage/DevIdePage.tsx index 027a7d5..9763303 100644 --- a/app/web/src/pages/DevIdePage/DevIdePage.js +++ b/app/web/src/pages/DevIdePage/DevIdePage.tsx @@ -1,10 +1,18 @@ import { createContext } from 'react' import Seo from 'src/components/Seo/Seo' -import IdeWrapper from 'src/components/IdeWrapper' +import IdeWrapper from 'src/components/IdeWrapper/IdeWrapper' import { Toaster } from '@redwoodjs/web/toast' -import { useIdeState } from 'src/helpers/hooks/useIdeState' +import { useIdeState, State, initialState } from 'src/helpers/hooks/useIdeState' -export const IdeContext = createContext() +interface IdeContextType { + state: State + thunkDispatch: (actionOrThunk: any) => any +} + +export const IdeContext = createContext({ + state: initialState, + thunkDispatch: () => {}, +}) const DevIdePage = ({ cadPackage }) => { const [state, thunkDispatch] = useIdeState() return (