Merge pull request #78 from kc8apf/docker_db_builder

Docker-based db builder

.gcloudignore

diff --git a/.gcloudignore b/.gcloudignore
new file mode 100644
index 0000000..ef2ea2e
--- /dev/null
+++ b/.gcloudignore
@@ -0,0 +1,15 @@
+# This file specifies files that are *not* uploaded to Google Cloud Platform
+# using gcloud. It follows the same syntax as .gitignore, with the addition of
+# "#!include" directives (which insert the entries of the given .gitignore-style
+# file at that point).
+#
+# For more information, run:
+#   $ gcloud topic gcloudignore
+#
+.gcloudignore
+#!include:.gitignore
+
+# Don't bother uploading files not needed to generate a database.
+experiments
+minitests
+vagrant

.gitignore

diff --git a/.gitignore b/.gitignore
index 05e207b..28f1075 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,3 +1,5 @@
 /build/
 .Xil
-database/*
+# Ignore database directories _except_ for their settings
+database/*/*
+!database/*/settings.sh

Dockerfile

diff --git a/Dockerfile b/Dockerfile
new file mode 100644
index 0000000..a254495
--- /dev/null
+++ b/Dockerfile
@@ -0,0 +1,12 @@
+ARG DEV_ENV_IMAGE
+
+FROM ${DEV_ENV_IMAGE} AS db_builder
+ARG NUM_PARALLEL_JOBS=1
+
+COPY . /source
+RUN cd /source && make -j${NUM_PARALLEL_JOBS} database
+RUN find /source/database -mindepth 1 -maxdepth 1 -type d -exec /source/htmlgen/htmlgen.py --settings={}/settings.sh --output=/output/html \;
+RUN mkdir -p /output/raw && find /source/database -mindepth 1 -maxdepth 1 -type d -exec cp -R {} /output/raw \;
+
+FROM nginx:alpine
+COPY --from=db_builder /output /usr/share/nginx/html

Makefile

diff --git a/Makefile b/Makefile
index 297b666..2028b5d 100644
--- a/Makefile
+++ b/Makefile
@@ -1,13 +1,21 @@
-JOBS ?= $(shell nproc)
-JOBS ?= 2
 CLANG_FORMAT ?= clang-format
 
-go:
+.PHONY: database format clean
+
+build:
 	git submodule update --init --recursive
 	mkdir -p build
-	cd build; cmake ..; make -j$(JOBS)
+	cd build; cmake ..; $(MAKE)
+
+database: build
+	$(MAKE) -C $@
 
 format:
 	find . -name \*.cc -and -not -path './third_party/*' -and -not -path './.git/*' -exec $(CLANG_FORMAT) -style=file -i {} \;
 	find . -name \*.h -and -not -path './third_party/*' -and -not -path './.git/*' -exec $(CLANG_FORMAT) -style=file -i {} \;
 	find . -name \*.py -and -not -path './third_party/*' -and -not -path './.git/*' -exec yapf -p -i {} \;
+
+clean:
+	$(MAKE) -C database clean
+	$(MAKE) -C fuzzers clean
+	rm -rf build

cloudbuild.yaml

diff --git a/cloudbuild.yaml b/cloudbuild.yaml
new file mode 100644
index 0000000..c3f12bc
--- /dev/null
+++ b/cloudbuild.yaml
@@ -0,0 +1,21 @@
+steps:
+- name: 'gcr.io/cloud-builders/docker'
+  args:
+    - 'build'
+    - '--build-arg'
+    - 'DEV_ENV_IMAGE=${_DEV_ENV_IMAGE}'
+    - '--build-arg'
+    - 'NUM_PARALLEL_JOBS=${_NUM_CPUS}'
+    - '-t'
+    - '${_GCR_ZONE}/$PROJECT_ID/${_IMAGE_NAME}:${TAG_NAME}'
+    - '.'
+options:
+  disk_size_gb: 1000
+  machine_type: N1_HIGHCPU_32
+timeout: '21600s'
+substitutions:
+  _GCR_ZONE: 'gcr.io'
+  _IMAGE_NAME: 'prjxray-db'
+  _NUM_CPUS: '16'
+images:
+  - '${_GCR_ZONE}/$PROJECT_ID/${_IMAGE_NAME}:${TAG_NAME}'

database/Makefile

diff --git a/database/Makefile b/database/Makefile
new file mode 100644
index 0000000..99fab40
--- /dev/null
+++ b/database/Makefile
@@ -0,0 +1,10 @@
+.NOTPARALLEL:
+
+SUBDIRS := $(patsubst %/,%, $(wildcard */))
+
+.PHONY: $(SUBDIRS)
+
+$(MAKECMDGOALS): $(SUBDIRS)
+
+$(SUBDIRS):
+	$(MAKE) -C $@ -f ../Makefile.database $(MAKECMDGOALS)

database/Makefile.database

diff --git a/database/Makefile.database b/database/Makefile.database
new file mode 100644
index 0000000..2e40796
--- /dev/null
+++ b/database/Makefile.database
@@ -0,0 +1,15 @@
+.PHONY: all clean
+
+all: tilegrid.json
+
+# Small dance to say that there is a single recipe that is run once to generate
+# multiple files.
+tileconn.json tilegrid.json: fuzzers
+.INTERMEDIATE: fuzzers
+fuzzers: SHELL:=/bin/bash
+fuzzers: settings.sh
+	source settings.sh && $(MAKE) -C ../../fuzzers all
+
+clean:
+	rm -f *.db tileconn.json tilegrid.json *.yaml
+	rm -rf html