Merge branch 'develop'

.gitignore

@@ -60,3 +60,6 @@ target/
 
 # Pycharm
 .idea/
+
+# MacOS
+*.DS_Store

README.md

@@ -1,5 +1,15 @@
 # datasketch-io
 
+
+## Setting your Docker secrets
+
+The docker compose file for the Datasketch stack requires that the `pg_passwd_datasketch` and `datasketch_key` secret be set before deployment.
+
+You will need to ssh into the node manager and create these secrets:
+```
+echo "mysecret" | docker secret create <secret_name> -
+```
+
 ## Build docker image
 
 `docker build -t androiddrew/datasketch:latest -f ./services/cms/Dockerfile ./services/cms`
@@ -33,3 +43,11 @@ host    replication     all             127.0.0.1/32            trust
 host    replication     all             ::1/128                 trust
 host    datasketch      datasketch      host.docker.internal    trust
 ```
+
+## Trouble shooting
+
+You can run the following command to get the logs for a task, running or not.
+
+```
+docker logs $(docker inspect --format "{{.Status.ContainerStatus.ContainerID}}" <task_id>)
+```

config/nginx/conf.d/datasketch.http.conf

@@ -0,0 +1,7 @@
+# Datasketch.io
+
+upstream datasketch_app {
+    server swarm1.androiddrew.com:8000;
+    server swarm2.androiddrew.com:8000;
+    server swarm3.androiddrew.com:8000;
+}

config/nginx/sites-available/datasketch.io

@@ -0,0 +1,40 @@
+# Datasketch.io
+# Upstream set datasketch_app in /etc/nginx/conf.d/datasketch.http.conf
+
+server {
+    # HTTP CONFIG
+    server_name datasketch.io;
+
+
+    root /var/www/datasketch;
+    # index index.html;
+
+    location  / {
+
+        proxy_pass http://datasketch_app;
+        proxy_set_header	Host	$host;
+		proxy_set_header	X-Forwarded-For	$remote_addr;
+	    # try_files $uri $uri/ =404;
+	
+	}
+
+    listen 443 ssl; # managed by Certbot
+    ssl_certificate /etc/letsencrypt/live/datasketch.io/fullchain.pem; # managed by Certbot
+    ssl_certificate_key /etc/letsencrypt/live/datasketch.io/privkey.pem; # managed by Certbot
+    include /etc/letsencrypt/options-ssl-nginx.conf; # managed by Certbot
+    ssl_dhparam /etc/letsencrypt/ssl-dhparams.pem; # managed by Certbot
+
+}
+
+server {
+    if ($host = datasketch.io) {
+        return 301 https://$host$request_uri;
+    } # managed by Certbot
+
+
+        listen 80;
+        server_name datasketch.io;
+    return 404; # managed by Certbot
+
+
+}

docker-compose-swarm.yml

@@ -0,0 +1,70 @@
+version: '3.6'
+
+services:
+
+  cms:
+    image: androiddrew/datasketch:latest
+    deploy:
+      replicas: 1
+      restart_policy:
+        condition: on-failure
+    ports:
+      - 8000:8000
+    secrets:
+      - datasketch_key
+      - pg_passwd_datasketch
+    environment:
+      - DJANGO_SETTINGS_MODULE=cms.settings.production
+      - DB_HOST=db
+      - DB_USER=datasketch
+    volumes:
+      - datasketch-media-vol:/code/media
+    depends_on:
+      - db
+    networks:
+      - datasketch_net
+
+  db:
+    image: postgres:10.5-alpine
+    deploy:
+      replicas: 1
+      restart_policy:
+        condition: on-failure
+      # Docker secrets for postgres are referenced by /run/secrets/<secret_name>
+    secrets:
+      - pg_passwd_datasketch
+    environment:
+      - POSTGRES_USER=datasketch
+      - POSTGRES_PASSWORD_FILE=/run/secrets/pg_passwd_datasketch
+      - POSTGRES_DB=datasketch
+    expose:
+      - 5432
+    volumes:
+      - datasketch-db-vol:/var/lib/postgresql/data
+    networks:
+      - datasketch_net
+
+
+networks:
+  datasketch_net:
+    driver: overlay
+
+volumes:
+  datasketch-db-vol:
+    driver: local
+    driver_opts:
+      type: nfs
+      o: addr=nas1.androiddrew.com,rw
+      device: ":/volume1/expanse/datasketch_pg_data"
+  datasketch-media-vol:
+    driver: local
+    driver_opts:
+      type: nfs
+      o: addr=nas1.androiddrew.com,rw
+      device: ":/volume1/expanse/datasketch_media"
+
+secrets:
+  pg_passwd_datasketch:
+    external: true
+  datasketch_key:
+    external: true

services/cms/Dockerfile

@@ -2,7 +2,7 @@ FROM python:3.7
 LABEL maintainer="drew@androiddrew.comw"
 
 ENV PYTHONUNBUFFERED 1
-ENV DJANGO_ENV dev
+ENV DJANGO_ENV production
 
 COPY ./requirements.txt /code/requirements.txt
 RUN pip install -r /code/requirements.txt
@@ -11,7 +11,7 @@ RUN pip install gunicorn
 COPY . /code/
 WORKDIR /code/
 
-# RUN python manage.py migrate # removed to garuantee build
+RUN python manage.py collectstatic
 
 RUN useradd wagtail
 RUN chown -R wagtail /code

services/cms/README.md

@@ -0,0 +1,26 @@
+# Datasketch CMS
+
+This service is built using the [Wagtail](http://docs.wagtail.io/en/v2.3/) framework.
+
+## Middleware
+
+Since this application is being deployed to a Docker swarm it utilizes [Whitenoise](http://whitenoise.evans.io/en/stable/) and the Django specific whitenoise middleware to deliver static content. This solution is suitable for serving low traffic static content. To improve performance an Nginx reverse proxy could leverage HTTP caching or could be configured to serve all static content directly.
+
+## Environmental Variables
+
+The follow environment variables must be set for development. Note that the DB_PASSWD for production must be a docker secret.
+
+```
+DB_HOST=<hostname for database>
+DB_PASSWD=ec<Password for datasketch user>
+```
+
+In production you must set the `DJANGO_SETTINGS_MODULE` to the appropriate module:
+
+```
+DJANGO_SETTINGS_MODULE=cms.settings.production
+```
+
+## Migrations
+
+The `manage.py` file is used to upgrade the backend database. You will need to log into the container on the appropriate host and execute the command `python manage.py migrate` manually.

services/cms/cms/settings/base.py

@@ -48,6 +48,7 @@ INSTALLED_APPS = [
 ]
 
 MIDDLEWARE = [
+    "whitenoise.middleware.WhiteNoiseMiddleware",
     "django.contrib.sessions.middleware.SessionMiddleware",
     "django.middleware.common.CommonMiddleware",
     "django.middleware.csrf.CsrfViewMiddleware",

services/cms/cms/settings/production.py

@@ -2,6 +2,42 @@ from .base import *
 
 DEBUG = False
 
+# Configured with docker secrets
+SECRET_KEY = open("/run/secrets/datasketch_key", "r").read().strip()
+PG_PASSWD = open("/run/secrets/pg_passwd_datasketch", "r").read().strip()
+
+# SECURITY WARNING: define the correct hosts in production!
+ALLOWED_HOSTS = [".datasketch.io", ".androiddrew.com", "localhost", "127.0.0.1"]
+
+EMAIL_BACKEND = "django.core.mail.backends.console.EmailBackend"
+
+DATABASES = {
+    "default": {
+        "ENGINE": "django.db.backends.postgresql",
+        "NAME": "datasketch",
+        "USER": "datasketch",
+        "PASSWORD": PG_PASSWD,
+        "HOST": os.environ.get("DB_HOST") if os.environ.get("DB_HOST") else "127.0.0.1",
+        "PORT": "5432",
+    }
+}
+
+# Set as the volume map for media on the nas
+MEDIA_URL = "/code/media/"
+
+LOGGING = {
+    "version": 1,
+    "disable_exisiing_loggers": False,
+    "handlers": {"console": {"class": "logging.StreamHandler"}},
+    "loggers": {
+        "django.request": {
+            "handlers": ["console"],
+            "level": os.getenv("DJANGO_LOG_LEVEL", "INFO"),
+        }
+    },
+}
+
+
 try:
     from .local import *
 except ImportError:

services/cms/requirements.txt

@@ -1,3 +1,4 @@
 Django>=2.1,<2.2
 wagtail>=2.3,<2.4
 psycopg2>=2.7.5
+whitenoise>=4.1