Initial work to support curv (#578)

* Initial work to support curv

* Correct the initial code file location

* Preview and stl mvp working

* Prepare changes for review and preview build

* Run curv inside of /tmp

When exporting an stl it writes temporary files which is not allowed
when deployed to aws unless it's in temp.

* Lock in specific curv commit for reproducible builds

see: https://discord.com/channels/412182089279209474/886321278821216277/912507472441401385

* Add curv to backend schema

* Frontend changes to accommodate curv deploy

* Use vcount instead of vsize, as it's independant of geometry size,

This is good for CadHub usecase where we don't know anything about the
user's project

* Final tweaks for deploy

virtual screen size does matter,and curv is a little more memory hungry
than the other functions

* Format project

Co-authored-by: lf94 <inbox@leefallat.ca>
Co-authored-by: Kurt Hutten <k.hutten@protonmail.ch>

app/api/db/migrations/20211129205924_curv/migration.sql

@@ -0,0 +1,2 @@
+-- AlterEnum
+ALTER TYPE "CadPackage" ADD VALUE 'curv';

app/api/db/schema.prisma

@@ -37,7 +37,8 @@ model User {
 enum CadPackage {
   openscad
   cadquery
-  jscad // TODO #422, add jscad to db schema when were ready to enable saving of jscad projects
+  jscad
+  curv
 }
 
 model Project {

app/api/src/docker/aws-emulator.js

@@ -34,8 +34,12 @@ const makeRequest = (route, port) => [
 
 app.post(...makeRequest('/openscad/preview', 5052))
 app.post(...makeRequest('/openscad/stl', 5053))
+
 app.post(...makeRequest('/cadquery/stl', 5060))
 
+app.post(...makeRequest('/curv/preview', 5070))
+app.post(...makeRequest('/curv/stl', 5071))
+
 app.listen(port, () => {
   console.log(`Example app listening at http://localhost:${port}`)
 })

app/api/src/docker/cadquery/cadquery.ts

@@ -9,12 +9,16 @@ const stl = async (req, _context, callback) => {
   console.log('eventBody', eventBody)
 
   const { file, settings } = JSON.parse(eventBody)
-  const { error, consoleMessage, fullPath } = await runCQ({ file, settings })
+  const { error, consoleMessage, fullPath, tempFile } = await runCQ({
+    file,
+    settings,
+  })
   await storeAssetAndReturnUrl({
     error,
     callback,
     fullPath,
     consoleMessage,
+    tempFile,
   })
 }
 

app/api/src/docker/cadquery/runCQ.ts

@@ -53,7 +53,7 @@ export const runCQ = async ({
       15000,
       true
     )
-    return { consoleMessage, fullPath }
+    return { consoleMessage, fullPath, tempFile }
   } catch (error) {
     return { error: consoleMessage || error, fullPath }
   }

app/api/src/docker/common/utils.ts

@@ -104,11 +104,13 @@ export async function storeAssetAndReturnUrl({
   callback,
   fullPath,
   consoleMessage,
+  tempFile,
 }: {
   error: string
   callback: Function
   fullPath: string
   consoleMessage: string
+  tempFile: string
 }) {
   if (error) {
     const response = {
@@ -124,6 +126,7 @@ export async function storeAssetAndReturnUrl({
 
     try {
       buffer = await readFile(fullPath, { encoding: 'base64' })
+      await runCommand(`rm -R /tmp/${tempFile}`)
     } catch (e) {
       console.log('read file error', e)
       const response = {

app/api/src/docker/curv/Dockerfile

@@ -0,0 +1,67 @@
+FROM public.ecr.aws/lts/ubuntu:20.04_stable
+
+ARG DEBIAN_FRONTEND=noninteractive
+
+
+RUN apt-get update --fix-missing -qq
+RUN apt-get update --fix-missing && apt-get -y -qq install software-properties-common dirmngr apt-transport-https lsb-release ca-certificates xvfb
+
+RUN apt-get update -qq
+
+RUN apt-get -y -qq install git \
+    software-properties-common \
+    xvfb unzip maim clang cmake \
+    git-core libboost-all-dev \
+    libopenexr-dev libtbb-dev \
+    libglm-dev libpng-dev \
+    libeigen3-dev dbus-x11 \
+    libxcursor-dev libxinerama-dev \
+    libxrandr-dev libglu1-mesa-dev \
+    libgles2-mesa-dev libgl1-mesa-dev \
+    libxi-dev
+
+# Use commit to make sure build is reproduceable
+RUN git clone --recursive https://github.com/curv3d/curv && \
+    cd curv && \
+    git checkout b849eb57fba121f9f218dc065dc1f5ebc619836d && \
+    make && make install
+
+# install node14, see comment at the top of node14source_setup.sh
+ADD src/docker/common/node14source_setup.sh /nodesource_setup.sh
+RUN ["chmod", "+x", "/nodesource_setup.sh"]
+RUN bash nodesource_setup.sh
+RUN apt-get install -y nodejs
+
+# Install aws-lambda-cpp build dependencies, this is for the post install script in aws-lambda-ric (in package.json)
+RUN apt-get update && \
+    apt-get install -y \
+    g++ \
+    make \
+    cmake \
+    unzip \
+    automake autoconf libtool \
+    libcurl4-openssl-dev
+
+# Add the lambda emulator for local dev, (see entrypoint.sh for where it's used),
+# I have the file locally (gitignored) to speed up build times (as it downloads everytime),
+# but you can use the http version of the below ADD command or download it yourself from that url.
+ADD src/docker/common/aws-lambda-rie /usr/local/bin/aws-lambda-rie
+# ADD https://github.com/aws/aws-lambda-runtime-interface-emulator/releases/download/v1.0/aws-lambda-rie /usr/local/bin/aws-lambda-rie
+RUN ["chmod", "+x", "/usr/local/bin/aws-lambda-rie"]
+
+WORKDIR /var/task/
+COPY package*.json /var/task/
+RUN npm install
+RUN npm install aws-lambda-ric@1.0.0
+
+RUN echo "cadhub-concat-split" > /var/task/cadhub-concat-split
+
+# using built javascript from dist
+# run `yarn rw build` before bulding this image
+COPY dist/docker/curv/* /var/task/js/
+COPY dist/docker/common/* /var/task/common/
+COPY src/docker/common/entrypoint.sh /entrypoint.sh
+RUN ["chmod", "+x", "/entrypoint.sh"]
+
+ENTRYPOINT ["sh", "/entrypoint.sh"]
+CMD [ "js/curv.preview" ]

app/api/src/docker/curv/curv.ts

@@ -0,0 +1,48 @@
+import { runCurv, stlExport } from './runCurv'
+import middy from 'middy'
+import { cors } from 'middy/middlewares'
+import { loggerWrap, storeAssetAndReturnUrl } from '../common/utils'
+
+const preview = async (req, _context, callback) => {
+  _context.callbackWaitsForEmptyEventLoop = false
+  const eventBody = Buffer.from(req.body, 'base64').toString('ascii')
+  console.log('eventBody', eventBody)
+
+  const { file, settings } = JSON.parse(eventBody)
+  const { error, consoleMessage, fullPath, tempFile } = await runCurv({
+    file,
+    settings,
+  })
+  await storeAssetAndReturnUrl({
+    error,
+    callback,
+    fullPath,
+    consoleMessage,
+    tempFile,
+  })
+}
+
+const stl = async (req, _context, callback) => {
+  _context.callbackWaitsForEmptyEventLoop = false
+  const eventBody = Buffer.from(req.body, 'base64').toString('ascii')
+
+  console.log(eventBody, 'eventBody')
+
+  const { file, settings } = JSON.parse(eventBody)
+  const { error, consoleMessage, fullPath, tempFile } = await stlExport({
+    file,
+    settings,
+  })
+  await storeAssetAndReturnUrl({
+    error,
+    callback,
+    fullPath,
+    consoleMessage,
+    tempFile,
+  })
+}
+
+module.exports = {
+  stl: middy(loggerWrap(stl)).use(cors()),
+  preview: middy(loggerWrap(preview)).use(cors()),
+}

app/api/src/docker/curv/runCurv.ts

@@ -0,0 +1,114 @@
+import { writeFiles, runCommand } from '../common/utils'
+import { nanoid } from 'nanoid'
+
+export const runCurv = async ({
+  file,
+  settings: { size: { x = 500, y = 500 } = {}, parameters } = {}, // TODO add view settings
+} = {}): Promise<{
+  error?: string
+  consoleMessage?: string
+  fullPath?: string
+  customizerPath?: string
+  tempFile?: string
+}> => {
+  const tempFile = await writeFiles(
+    [
+      { file, fileName: 'main.curv' },
+      {
+        file: JSON.stringify({
+          parameterSets: { default: parameters },
+          fileFormatVersion: '1',
+        }),
+        fileName: 'params.json',
+      },
+    ],
+    'a' + nanoid() // 'a' ensure nothing funny happens if it start with a bad character like "-", maybe I should pick a safer id generator :shrug:
+  )
+  const fullPath = `/tmp/${tempFile}/output.gz`
+  const imPath = `/tmp/${tempFile}/output.png`
+  const customizerPath = `/tmp/${tempFile}/customizer.param`
+
+  const command = [
+    'xvfb-run --auto-servernum --server-args "-screen 0 3840x2160x24" curv',
+    `-o ${imPath}`,
+    `-O xsize=${x}`,
+    `-O ysize=${y}`,
+    `-O bg=webRGB[26,26,29]`, // #1A1A1D
+    `/tmp/${tempFile}/main.curv`,
+  ].join(' ')
+  console.log('command', command)
+
+  try {
+    const consoleMessage = await runCommand(command, 15000)
+    await writeFiles(
+      [
+        {
+          file: JSON.stringify({
+            consoleMessage,
+            type: 'png',
+          }),
+          fileName: 'metadata.json',
+        },
+      ],
+      tempFile
+    )
+    await runCommand(
+      `cat ${imPath} /var/task/cadhub-concat-split /tmp/${tempFile}/metadata.json | gzip > ${fullPath}`,
+      15000
+    )
+    return { consoleMessage, fullPath, customizerPath, tempFile }
+  } catch (dirtyError) {
+    return { error: dirtyError }
+  }
+}
+
+export const stlExport = async ({ file, settings: { parameters } } = {}) => {
+  const tempFile = await writeFiles(
+    [
+      { file, fileName: 'main.curv' },
+      {
+        file: JSON.stringify({
+          parameterSets: { default: parameters },
+          fileFormatVersion: '1',
+        }),
+        fileName: 'params.json',
+      },
+    ],
+    'a' + nanoid() // 'a' ensure nothing funny happens if it start with a bad character like "-", maybe I should pick a safer id generator :shrug:
+  )
+  const fullPath = `/tmp/${tempFile}/output.gz`
+  const stlPath = `/tmp/${tempFile}/output.stl`
+  const command = [
+    '(cd /tmp && curv',
+    '-o',
+    stlPath,
+    '-O jit',
+    '-O vcount=350000',
+    `/tmp/${tempFile}/main.curv`,
+    ')',
+  ].join(' ')
+
+  try {
+    // lambda will time out before this, we might need to look at background jobs if we do git integration stl generation
+    const consoleMessage = await runCommand(command, 60000)
+    await writeFiles(
+      [
+        {
+          file: JSON.stringify({
+            consoleMessage,
+            type: 'stl',
+          }),
+          fileName: 'metadata.json',
+        },
+      ],
+      tempFile
+    )
+    await runCommand(
+      `cat ${stlPath} /var/task/cadhub-concat-split /tmp/${tempFile}/metadata.json | gzip > ${fullPath} && rm ${stlPath}`,
+      15000
+    )
+    return { consoleMessage, fullPath, tempFile }
+  } catch (error) {
+    return { error, fullPath }
+  }
+}

app/api/src/docker/docker-compose.yml

@@ -45,3 +45,28 @@ services:
       AWS_ACCESS_KEY_ID: "${DEV_AWS_ACCESS_KEY_ID}"
       BUCKET: "${DEV_BUCKET}"
 
+  curv-preview:
+    build:
+      context: ../../
+      dockerfile: ./src/docker/curv/Dockerfile
+    image: curv
+    command: js/curv.preview
+    # Adding volumes so that the containers can be restarted for js only changes in local dev
+    volumes:
+      - ../../dist/docker/curv:/var/task/js/
+      - ../../dist/docker/common:/var/task/common/
+    ports:
+      - "5070:8080"
+  curv-stl:
+    build:
+      context: ../../
+      dockerfile: ./src/docker/curv/Dockerfile
+    image: curv
+    command: js/curv.stl
+    # Adding volumes so that the containers can be restarted for js only changes in local dev
+    volumes:
+      - ../../dist/docker/curv:/var/task/js/
+      - ../../dist/docker/common:/var/task/common/
+    ports:
+      - "5071:8080"
+

app/api/src/docker/openscad/openscad.ts

@@ -9,7 +9,7 @@ const preview = async (req, _context, callback) => {
   console.log('eventBody', eventBody)
 
   const { file, settings } = JSON.parse(eventBody)
-  const { error, consoleMessage, fullPath } = await runScad({
+  const { error, consoleMessage, fullPath, tempFile } = await runScad({
     file,
     settings,
   })
@@ -18,6 +18,7 @@ const preview = async (req, _context, callback) => {
     callback,
     fullPath,
     consoleMessage,
+    tempFile,
   })
 }
 
@@ -28,7 +29,7 @@ const stl = async (req, _context, callback) => {
   console.log(eventBody, 'eventBody')
 
   const { file, settings } = JSON.parse(eventBody)
-  const { error, consoleMessage, fullPath } = await stlExport({
+  const { error, consoleMessage, fullPath, tempFile } = await stlExport({
     file,
     settings,
   })
@@ -37,6 +38,7 @@ const stl = async (req, _context, callback) => {
     callback,
     fullPath,
     consoleMessage,
+    tempFile,
   })
 }
 

app/api/src/docker/openscad/runScad.ts

@@ -25,6 +25,7 @@ export const runScad = async ({
   consoleMessage?: string
   fullPath?: string
   customizerPath?: string
+  tempFile?: string
 }> => {
   const tempFile = await writeFiles(
     [
@@ -88,7 +89,7 @@ export const runScad = async ({
       `cat ${imPath} /var/task/cadhub-concat-split /tmp/${tempFile}/metadata.json | gzip > ${fullPath}`,
       15000
     )
-    return { consoleMessage, fullPath, customizerPath }
+    return { consoleMessage, fullPath, customizerPath, tempFile }
   } catch (dirtyError) {
     return { error: cleanOpenScadError(dirtyError) }
   }
@@ -143,7 +144,7 @@ export const stlExport = async ({ file, settings: { parameters } } = {}) => {
       `cat ${stlPath} /var/task/cadhub-concat-split /tmp/${tempFile}/metadata.json | gzip > ${fullPath}`,
       15000
     )
-    return { consoleMessage, fullPath, customizerPath }
+    return { consoleMessage, fullPath, customizerPath, tempFile }
   } catch (error) {
     return { error, fullPath }
   }

app/api/src/docker/serverless.yml

@@ -23,6 +23,9 @@ provider:
       cadqueryimage:
         path: ../../
         file: ./src/docker/cadquery/Dockerfile
+      curvimage:
+        path: ../../
+        file: ./src/docker/curv/Dockerfile
   apiGateway:
     metrics: true
     binaryMediaTypes:
@@ -84,6 +87,7 @@ functions:
     timeout: 30
     environment:
       BUCKET: cad-preview-bucket-prod-001
+
   cadquerystl:
     image:
       name: cadqueryimage
@@ -99,6 +103,34 @@ functions:
     timeout: 30
     environment:
       BUCKET: cad-preview-bucket-prod-001
+
+  curvpreview:
+    image:
+      name: curvimage
+      command:
+        - js/curv.preview
+      entryPoint:
+        - '/entrypoint.sh'
+    events:
+      - http:
+          path: curv/preview
+          method: post
+          cors: true
+    timeout: 25
+    memorySize: 3008
+  curvstl:
+    image:
+      name: curvimage
+      command:
+        - js/curv.stl
+      entryPoint:
+        - '/entrypoint.sh'
+    events:
+      - http:
+          path: curv/stl
+          method: post
+          cors: true
+    timeout: 30
 #    The following are a few example events you can configure
 #    NOTE: Please make sure to change your handler code to work with those events
 #    Check the event documentation for details

app/api/src/graphql/projects.sdl.ts

@@ -19,10 +19,12 @@ export const schema = gql`
     childForks: [Project]!
   }
 
+  # should match enum in api/db/schema.prisma
   enum CadPackage {
     openscad
     cadquery
     jscad
+    curv
   }
 
   type Query {